#include "X86ISelLowering.h"
#include "Utils/X86ShuffleDecode.h"
#include "X86CallingConv.h"
#include "X86InstrBuilder.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "X86TargetObjectFile.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/VariadicFunction.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <bitset>
#include <numeric>
#include <cctype>
#include "X86GenCallingConv.inc"

Data Structures
struct	IntrinsicData

Macros
#define	DEBUG_TYPE "x86-isel"

Enumerations
enum	StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn }

enum	IntrinsicType { GATHER, SCATTER, PREFETCH, RDSEED, RDRAND, RDPMC, RDTSC, XTEST }

Functions
	STATISTIC (NumTailCalls,"Number of tail calls")

static SDValue	getMOVL (SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1, SDValue V2)

static SDValue	ExtractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl, unsigned vectorWidth)

static SDValue	Extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl)

static SDValue	Extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl)
	Generate a DAG to grab 256-bits from a 512-bit vector. More...

static SDValue	InsertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl, unsigned vectorWidth)

static SDValue	Insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl)

static SDValue	Insert256BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl)

static SDValue	Concat128BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, SDLoc dl)

static SDValue	Concat256BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, SDLoc dl)

static TargetLoweringObjectFile *	createTLOF (const Triple &TT)

static void	getMaxByValAlign (Type *Ty, unsigned &MaxAlign)

static StructReturnType	callIsStructReturn (const SmallVectorImpl< ISD::OutputArg > &Outs)

static StructReturnType	argsAreStructReturn (const SmallVectorImpl< ISD::InputArg > &Ins)

static SDValue	CreateCopyOfByValArgument (SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, SDLoc dl)

static bool	IsTailCallConvention (CallingConv::ID CC)

static bool	IsCCallConvention (CallingConv::ID CC)
	Return true if the calling convention is a C calling convention. More...

static bool	FuncIsMadeTailCallSafe (CallingConv::ID CC, bool GuaranteedTailCallOpt)

static SDValue	EmitTailCallStoreRetAddr (SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT, unsigned SlotSize, int FPDiff, SDLoc dl)

static bool	MatchingStackOffset (SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo MFI, const MachineRegisterInfo MRI, const X86InstrInfo *TII)

static bool	MayFoldLoad (SDValue Op)

static bool	MayFoldIntoStore (SDValue Op)

static bool	isTargetShuffle (unsigned Opcode)

static SDValue	getTargetShuffleNode (unsigned Opc, SDLoc dl, EVT VT, SDValue V1, SelectionDAG &DAG)

static SDValue	getTargetShuffleNode (unsigned Opc, SDLoc dl, EVT VT, SDValue V1, unsigned TargetMask, SelectionDAG &DAG)

static SDValue	getTargetShuffleNode (unsigned Opc, SDLoc dl, EVT VT, SDValue V1, SDValue V2, unsigned TargetMask, SelectionDAG &DAG)

static SDValue	getTargetShuffleNode (unsigned Opc, SDLoc dl, EVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)

static bool	isX86CCUnsigned (unsigned X86CC)
	Return true if the condition is an unsigned comparison operation. More...

static unsigned	TranslateX86CC (ISD::CondCode SetCCOpcode, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)

static bool	hasFPCMov (unsigned X86CC)

static bool	isUndefOrInRange (int Val, int Low, int Hi)

static bool	isUndefOrEqual (int Val, int CmpVal)

static bool	isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low)

static bool	isPSHUFDMask (ArrayRef< int > Mask, MVT VT)

static bool	isPSHUFHWMask (ArrayRef< int > Mask, MVT VT, bool HasInt256)

static bool	isPSHUFLWMask (ArrayRef< int > Mask, MVT VT, bool HasInt256)

static bool	isPALIGNRMask (ArrayRef< int > Mask, MVT VT, const X86Subtarget *Subtarget)

static void	CommuteVectorShuffleMask (SmallVectorImpl< int > &Mask, unsigned NumElems)

static bool	isSHUFPMask (ArrayRef< int > Mask, MVT VT, bool Commuted=false)

static bool	isMOVHLPSMask (ArrayRef< int > Mask, MVT VT)

static bool	isMOVHLPS_v_undef_Mask (ArrayRef< int > Mask, MVT VT)

static bool	isMOVLPMask (ArrayRef< int > Mask, MVT VT)

static bool	isMOVLHPSMask (ArrayRef< int > Mask, MVT VT)

static bool	isINSERTPSMask (ArrayRef< int > Mask, MVT VT)

static SDValue	Compact8x32ShuffleNode (ShuffleVectorSDNode *SVOp, SelectionDAG &DAG)

static bool	isUNPCKLMask (ArrayRef< int > Mask, MVT VT, bool HasInt256, bool V2IsSplat=false)

static bool	isUNPCKHMask (ArrayRef< int > Mask, MVT VT, bool HasInt256, bool V2IsSplat=false)

static bool	isUNPCKL_v_undef_Mask (ArrayRef< int > Mask, MVT VT, bool HasInt256)

static bool	isUNPCKH_v_undef_Mask (ArrayRef< int > Mask, MVT VT, bool HasInt256)

static bool	isINSERT64x4Mask (ArrayRef< int > Mask, MVT VT, unsigned int *Imm)

static bool	isMOVLMask (ArrayRef< int > Mask, EVT VT)

static bool	isVPERM2X128Mask (ArrayRef< int > Mask, MVT VT, bool HasFp256)

static unsigned	getShuffleVPERM2X128Immediate (ShuffleVectorSDNode *SVOp)

static bool	isPermImmMask (ArrayRef< int > Mask, MVT VT, unsigned &Imm8)

static bool	isVPERMILPMask (ArrayRef< int > Mask, MVT VT)

static bool	isCommutedMOVLMask (ArrayRef< int > Mask, MVT VT, bool V2IsSplat=false, bool V2IsUndef=false)

static bool	isMOVSHDUPMask (ArrayRef< int > Mask, MVT VT, const X86Subtarget *Subtarget)

static bool	isMOVSLDUPMask (ArrayRef< int > Mask, MVT VT, const X86Subtarget *Subtarget)

static bool	isMOVDDUPYMask (ArrayRef< int > Mask, MVT VT, bool HasFp256)

static bool	isMOVDDUPMask (ArrayRef< int > Mask, MVT VT)

static bool	isVEXTRACTIndex (SDNode *N, unsigned vecWidth)

static bool	isVINSERTIndex (SDNode *N, unsigned vecWidth)

static unsigned	getShuffleSHUFImmediate (ShuffleVectorSDNode *N)

static unsigned	getShufflePSHUFHWImmediate (ShuffleVectorSDNode *N)

static unsigned	getShufflePSHUFLWImmediate (ShuffleVectorSDNode *N)

static unsigned	getShufflePALIGNRImmediate (ShuffleVectorSDNode *SVOp)

static unsigned	getExtractVEXTRACTImmediate (SDNode *N, unsigned vecWidth)

static unsigned	getInsertVINSERTImmediate (SDNode *N, unsigned vecWidth)

static bool	isZero (SDValue V)
	isZero - Returns true if Elt is a constant integer zero More...

static bool	ShouldXformToMOVHLPS (ArrayRef< int > Mask, MVT VT)

static bool	isScalarLoadToVector (SDNode N, LoadSDNode *LD=nullptr)

static bool	WillBeConstantPoolLoad (SDNode *N)

static bool	ShouldXformToMOVLP (SDNode V1, SDNode V2, ArrayRef< int > Mask, MVT VT)

static bool	isZeroShuffle (ShuffleVectorSDNode *N)

static SDValue	getZeroVector (EVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG, SDLoc dl)

static SDValue	getOnesVector (MVT VT, bool HasInt256, SelectionDAG &DAG, SDLoc dl)

static void	NormalizeMask (SmallVectorImpl< int > &Mask, unsigned NumElems)

static SDValue	getUnpackl (SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, SDValue V2)
	getUnpackl - Returns a vector_shuffle node for an unpackl operation. More...

static SDValue	getUnpackh (SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, SDValue V2)
	getUnpackh - Returns a vector_shuffle node for an unpackh operation. More...

static SDValue	PromoteSplati8i16 (SDValue V, SelectionDAG &DAG, int &EltNo)

static SDValue	getLegalSplat (SelectionDAG &DAG, SDValue V, int EltNo)
	getLegalSplat - Generate a legal splat with supported x86 shuffles More...

static SDValue	PromoteSplat (ShuffleVectorSDNode *SV, SelectionDAG &DAG)
	PromoteSplat - Splat is promoted to target supported vector shuffles. More...

static SDValue	getShuffleVectorZeroOrUndef (SDValue V2, unsigned Idx, bool IsZero, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static bool	getTargetShuffleMask (SDNode *N, MVT VT, SmallVectorImpl< int > &Mask, bool &IsUnary)

static SDValue	getShuffleScalarElt (SDNode *N, unsigned Index, SelectionDAG &DAG, unsigned Depth)

static unsigned	getNumOfConsecutiveZeros (ShuffleVectorSDNode *SVOp, unsigned NumElems, bool ZerosFromLeft, SelectionDAG &DAG, unsigned PreferredNum=-1U)

static bool	isShuffleMaskConsecutive (ShuffleVectorSDNode *SVOp, unsigned MaskI, unsigned MaskE, unsigned OpIdx, unsigned NumElems, unsigned &OpNum)

static bool	isVectorShiftRight (ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, bool &isLeft, SDValue &ShVal, unsigned &ShAmt)

static bool	isVectorShiftLeft (ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, bool &isLeft, SDValue &ShVal, unsigned &ShAmt)

static bool	isVectorShift (ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, bool &isLeft, SDValue &ShVal, unsigned &ShAmt)

static SDValue	LowerBuildVectorv16i8 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI)

static SDValue	LowerBuildVectorv8i16 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI)

static SDValue	LowerBuildVectorv4x32 (SDValue Op, unsigned NumElems, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI)
	LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32. More...

static SDValue	getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, SDLoc dl)

static SDValue	LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG)

static SDValue	EltsFromConsecutiveLoads (EVT VT, SmallVectorImpl< SDValue > &Elts, SDLoc &DL, SelectionDAG &DAG, bool isAfterLegalize)

static SDValue	LowerVectorBroadcast (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static int	getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx)
	For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. More...

static SDValue	buildFromShuffleMostly (SDValue Op, SelectionDAG &DAG)

static bool	isHorizontalBinOp (const BuildVectorSDNode *N, unsigned Opcode, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1)
	Return true if `N` implements a horizontal binop and return the operands for the horizontal binop into V0 and V1. More...

static SDValue	ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, SDLoc DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI)
	Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. More...

static SDValue	matchAddSub (const BuildVectorSDNode BV, SelectionDAG &DAG, const X86Subtarget Subtarget)
	Try to fold a build_vector that performs an 'addsub' into the sequence of 'vadd + vsub + blendi'. More...

static SDValue	PerformBUILD_VECTORCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)

static SDValue	LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG)

static bool	isNoopShuffleMask (ArrayRef< int > Mask)
	Tiny helper function to identify a no-op mask. More...

static bool	isSingleInputShuffleMask (ArrayRef< int > Mask)
	Helper function to classify a mask as a single-input mask. More...

static SDValue	getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, SelectionDAG &DAG)
	Get a 4-lane 8-bit shuffle immediate for a mask. More...

static SDValue	lowerV2F64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Handle lowering of 2-lane 64-bit floating point shuffles. More...

static SDValue	lowerV2I64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Handle lowering of 2-lane 64-bit integer shuffles. More...

static SDValue	lowerV4F32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Lower 4-lane 32-bit floating point shuffles. More...

static SDValue	lowerV4I32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Lower 4-lane i32 vector shuffles. More...

static SDValue	lowerV8I16SingleInputVectorShuffle (SDLoc DL, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. More...

static bool	shouldLowerAsInterleaving (ArrayRef< int > Mask)
	Detect whether the mask pattern should be lowered through interleaving. More...

static SDValue	lowerV8I16BasicBlendVectorShuffle (SDLoc DL, SDValue V1, SDValue V2, MutableArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Blend two v8i16 vectors using a naive unpack strategy. More...

static SDValue	lowerV8I16VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Generic lowering of 8-lane i16 shuffles. More...

static SDValue	lowerV16I8VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Generic lowering of v16i8 shuffles. More...

static SDValue	lower128BitVectorShuffle (SDValue Op, SDValue V1, SDValue V2, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Dispatching routine to lower various 128-bit x86 vector shuffles. More...

static bool	areAdjacentMasksSequential (ArrayRef< int > Mask)
	Tiny helper function to test whether adjacent masks are sequential. More...

static SDValue	lowerVectorShuffle (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
	Top-level lowering for x86 vector shuffles. More...

static bool	isBlendMask (ArrayRef< int > MaskVals, MVT VT, bool hasSSE41, bool hasInt256, unsigned *MaskOut=nullptr)

static SDValue	LowerVECTOR_SHUFFLEtoBlend (ShuffleVectorSDNode SVOp, unsigned MaskValue, const X86Subtarget Subtarget, SelectionDAG &DAG)

static bool	ShuffleCrosses128bitLane (MVT VT, unsigned InputIdx, unsigned OutputIdx)

static SDValue	getPSHUFB (ArrayRef< int > MaskVals, SDValue V1, SDLoc &dl, SelectionDAG &DAG)

static SDValue	LowerVECTOR_SHUFFLEv8i16 (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerVECTOR_SHUFFLEv16i16 (SDValue Op, SelectionDAG &DAG)
	v16i16 shuffles More...

static SDValue	LowerVECTOR_SHUFFLEv16i8 (ShuffleVectorSDNode SVOp, const X86Subtarget Subtarget, SelectionDAG &DAG)

static SDValue	LowerVECTOR_SHUFFLEv32i8 (ShuffleVectorSDNode SVOp, const X86Subtarget Subtarget, SelectionDAG &DAG)

static SDValue	RewriteAsNarrowerShuffle (ShuffleVectorSDNode *SVOp, SelectionDAG &DAG)

static SDValue	getVZextMovL (MVT VT, MVT OpVT, SDValue SrcOp, SelectionDAG &DAG, const X86Subtarget *Subtarget, SDLoc dl)

static SDValue	LowerVECTOR_SHUFFLE_256 (ShuffleVectorSDNode *SVOp, SelectionDAG &DAG)

static SDValue	LowerVECTOR_SHUFFLE_128v4 (ShuffleVectorSDNode *SVOp, SelectionDAG &DAG)

static bool	MayFoldVectorLoad (SDValue V)

static SDValue	getMOVDDup (SDValue &Op, SDLoc &dl, SDValue V1, SelectionDAG &DAG)

static SDValue	getMOVLowToHigh (SDValue &Op, SDLoc &dl, SelectionDAG &DAG, bool HasSSE2)

static SDValue	getMOVHighToLow (SDValue &Op, SDLoc &dl, SelectionDAG &DAG)

static SDValue	getMOVLP (SDValue &Op, SDLoc &dl, SelectionDAG &DAG, bool HasSSE2)

static SDValue	NarrowVectorLoadToElement (LoadSDNode *Load, unsigned Index, SelectionDAG &DAG)

static SDValue	getINSERTPS (ShuffleVectorSDNode *SVOp, SDLoc &dl, SelectionDAG &DAG)

static SDValue	LowerVectorIntExtend (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	NormalizeVectorShuffle (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static bool	BUILD_VECTORtoBlendMask (BuildVectorSDNode *BuildVector, unsigned &MaskValue)

static SDValue	LowerVSELECTtoBlend (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerINSERT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerSCALAR_TO_VECTOR (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode GA, SDValue InFlag, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false)

static SDValue	LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)

static SDValue	LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)

static SDValue	LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool is64Bit)

static SDValue	LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)

static SDValue	LowerShiftParts (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget)

static SDValue	LowerZERO_EXTEND_AVX512 (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerANY_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerZERO_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerFP_EXTEND (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerFABS (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerFNEG (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerFGETSIGN (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerVectorAllZeroTest (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static bool	hasNonFlagsUse (SDValue Op)
	return true if `Op` has a use that doesn't just read flags. More...

static bool	isAllOnes (SDValue V)

static int	translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1)
	Turns an ISD::CondCode into a value suitable for SSE floating point mask CMPs. More...

static SDValue	Lower256IntVSETCC (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget)

static SDValue	ChangeVSETULTtoVSETULE (SDLoc dl, SDValue Op1, SelectionDAG &DAG)
	Try to turn a VSETULT into a VSETULE by modifying its second operand `Op1`. If non-trivial (for example because it's not constant) return an empty value. More...

static SDValue	LowerVSETCC (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static bool	isX86LogicalCmp (SDValue Op)

static bool	isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG)

static SDValue	LowerSIGN_EXTEND_AVX512 (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerSIGN_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static bool	isAndOrOfSetCCs (SDValue Op, unsigned &Opc)

static bool	isXor1OfSetCC (SDValue Op)

static SDValue	LowerVACOPY (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	getTargetVShiftByConstNode (unsigned Opc, SDLoc dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)

static SDValue	getTargetVShiftNode (unsigned Opc, SDLoc dl, MVT VT, SDValue SrcOp, SDValue ShAmt, SelectionDAG &DAG)

static SDValue	LowerINTRINSIC_WO_CHAIN (SDValue Op, SelectionDAG &DAG)

static SDValue	getGatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget *Subtarget)

static SDValue	getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain)

static SDValue	getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain)

static void	getReadPerformanceCounter (SDNode N, SDLoc DL, SelectionDAG &DAG, const X86Subtarget Subtarget, SmallVectorImpl< SDValue > &Results)

static void	getReadTimeStampCounter (SDNode N, SDLoc DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget Subtarget, SmallVectorImpl< SDValue > &Results)

static SDValue	LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static void	InitIntinsicsMap ()

static SDValue	LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerCTLZ (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerCTLZ_ZERO_UNDEF (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerCTTZ (SDValue Op, SelectionDAG &DAG)

static SDValue	Lower256IntArith (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerADD (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerSUB (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerMUL (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerMUL_LOHI (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerScalarImmediateShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget)

static SDValue	LowerScalarVariableShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget)

static SDValue	LowerShift (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerXALUO (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerATOMIC_FENCE (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerCMP_SWAP (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerBITCAST (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static SDValue	LowerLOAD_SUB (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerADDC_ADDE_SUBC_SUBE (SDValue Op, SelectionDAG &DAG)

static SDValue	LowerFSINCOS (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)

static void	ReplaceATOMIC_LOAD (SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)

static MachineBasicBlock *	EmitXBegin (MachineInstr MI, MachineBasicBlock MBB, const TargetInstrInfo *TII)
	Utility function to emit xbegin specifying the start of an RTM region. More...

static MachineBasicBlock *	EmitPCMPSTRM (MachineInstr MI, MachineBasicBlock BB, const TargetInstrInfo *TII)

static MachineBasicBlock *	EmitPCMPSTRI (MachineInstr MI, MachineBasicBlock BB, const TargetInstrInfo *TII)

static MachineBasicBlock *	EmitMonitor (MachineInstr MI, MachineBasicBlock BB, const TargetInstrInfo TII, const X86Subtarget Subtarget)

static bool	checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock BB, const TargetRegisterInfo TRI)

static bool	isShuffleHigh128VectorInsertLow (ShuffleVectorSDNode *SVOp)

static bool	isShuffleLow128VectorInsertHigh (ShuffleVectorSDNode *SVOp)

static SDValue	PerformShuffleCombine256 (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)
	PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors. More...

static SmallVector< int, 4 >	getPSHUFShuffleMask (SDValue N)
	Get the PSHUF-style mask from PSHUF node. More...

static bool	combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
	Search for a combinable shuffle across a chain ending in pshufd. More...

static bool	combineRedundantHalfShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
	Search for a combinable shuffle across a chain ending in pshuflw or pshufhw. More...

static SDValue	PerformTargetShuffleCombine (SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
	Try to combine x86 target specific shuffles. More...

static SDValue	PerformShuffleCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)
	PerformShuffleCombine - Performs several different shuffle combines. More...

static SDValue	PerformTruncateCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	XFormVExtractWithShuffleIntoLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)

static SDValue	PerformEXTRACT_VECTOR_ELTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)

static std::pair< unsigned, bool >	matchIntegerMINMAX (SDValue Cond, EVT VT, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const X86Subtarget *Subtarget)
	Matches a VSELECT onto min/max or return 0 if the node doesn't match. More...

static SDValue	TransformVSELECTtoBlendVECTOR_SHUFFLE (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)

static SDValue	PerformSELECTCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC)

static SDValue	PerformCMOVCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)
	Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. More...

static SDValue	PerformINTRINSIC_WO_CHAINCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)

static SDValue	PerformMulCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)

static SDValue	PerformSHLCombine (SDNode *N, SelectionDAG &DAG)

static SDValue	performShiftToAllZeros (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)
	Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits. More...

static SDValue	PerformShiftCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)
	PerformShiftCombine - Combine shifts. More...

static SDValue	CMPEQCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static bool	CanFoldXORWithAllOnes (const SDNode *N)

static SDValue	WidenMaskArithmetic (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	PerformAndCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	PerformOrCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	performIntegerAbsCombine (SDNode *N, SelectionDAG &DAG)

static SDValue	PerformXorCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	PerformLOADCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)
	PerformLOADCombine - Do target-specific dag combines on LOAD nodes. More...

static SDValue	PerformSTORECombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)
	PerformSTORECombine - Do target-specific dag combines on STORE nodes. More...

static bool	isHorizontalBinOp (SDValue &LHS, SDValue &RHS, bool IsCommutative)

static SDValue	PerformFADDCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)
	PerformFADDCombine - Do target-specific dag combines on floating point adds. More...

static SDValue	PerformFSUBCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)
	PerformFSUBCombine - Do target-specific dag combines on floating point subs. More...

static SDValue	PerformFORCombine (SDNode *N, SelectionDAG &DAG)

static SDValue	PerformFMinFMaxCombine (SDNode *N, SelectionDAG &DAG)

static SDValue	PerformFANDCombine (SDNode *N, SelectionDAG &DAG)
	PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. More...

static SDValue	PerformFANDNCombine (SDNode *N, SelectionDAG &DAG)
	PerformFANDNCombine - Do target-specific dag combines on X86ISD::FANDN nodes. More...

static SDValue	PerformBTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)

static SDValue	PerformVZEXT_MOVLCombine (SDNode *N, SelectionDAG &DAG)

static SDValue	PerformSIGN_EXTEND_INREGCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)

static SDValue	PerformSExtCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	PerformFMACombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)

static SDValue	PerformZExtCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	PerformISDSETCCCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)

static SDValue	PerformINSERTPSCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)

static SDValue	MaterializeSETB (SDLoc DL, SDValue EFLAGS, SelectionDAG &DAG, MVT VT)

static SDValue	PerformSETCCCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	PerformBrCondCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)

static SDValue	performVectorCompareAndMaskUnaryOpCombine (SDNode *N, SelectionDAG &DAG)

static SDValue	PerformSINT_TO_FPCombine (SDNode N, SelectionDAG &DAG, const X86TargetLowering XTLI)

static SDValue	PerformADCCombine (SDNode *N, SelectionDAG &DAG, X86TargetLowering::DAGCombinerInfo &DCI)

static SDValue	OptimizeConditionalInDecrement (SDNode *N, SelectionDAG &DAG)

static SDValue	PerformAddCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)
	PerformADDCombine - Do target-specific dag combines on integer adds. More...

static SDValue	PerformSubCombine (SDNode N, SelectionDAG &DAG, const X86Subtarget Subtarget)

static SDValue	performVZEXTCombine (SDNode N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget Subtarget)
	performVZEXTCombine - Performs build vector combines More...

static bool	clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces)

Variables
static cl::opt< bool >	ExperimentalVectorWideningLegalization ("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening ""rather than promotion."), cl::Hidden)

static cl::opt< bool >	ExperimentalVectorShuffleLowering ("x86-experimental-vector-shuffle-lowering", cl::init(false), cl::desc("Enable an experimental vector shuffle lowering code path."), cl::Hidden)

std::map< unsigned, IntrinsicData >	IntrMap

Macro Definition Documentation

#define DEBUG_TYPE "x86-isel"

Definition at line 57 of file X86ISelLowering.cpp.

Enumeration Type Documentation

enum IntrinsicType

Enumerator
GATHER
SCATTER
PREFETCH
RDSEED
RDRAND
RDPMC
RDTSC
XTEST

Definition at line 14385 of file X86ISelLowering.cpp.

                    {
   GATHER, SCATTER, PREFETCH, RDSEED, RDRAND, RDPMC, RDTSC, XTEST
 };

enum StructReturnType

CallIsStructReturn - Determines whether a call uses struct return semantics.

Enumerator
NotStructReturn
RegStructReturn
StackStructReturn

Definition at line 2122 of file X86ISelLowering.cpp.

                       {
   NotStructReturn,
   RegStructReturn,
   StackStructReturn
 };

Function Documentation

static bool areAdjacentMasksSequential ( ArrayRef< int > Mask )

static

Tiny helper function to test whether adjacent masks are sequential.

Definition at line 7897 of file X86ISelLowering.cpp.

                                                            {
   for (int i = 0, Size = Mask.size(); i < Size; i += 2)
     if (Mask[i] + 1 != Mask[i+1])
       return false;
 
   return true;
 }

static StructReturnType argsAreStructReturn ( const SmallVectorImpl< ISD::InputArg > & Ins )

static

ArgsAreStructReturn - Determines whether a function uses struct return semantics.

Definition at line 2143 of file X86ISelLowering.cpp.

                                                              {
   if (Ins.empty())
     return NotStructReturn;
 
   const ISD::ArgFlagsTy &Flags = Ins[0].Flags;
   if (!Flags.isSRet())
     return NotStructReturn;
   if (Flags.isInReg())
     return RegStructReturn;
   return StackStructReturn;
 }

static bool BUILD_VECTORtoBlendMask	(	BuildVectorSDNode *	BuildVector,
		unsigned &	MaskValue
	)

static

Definition at line 9608 of file X86ISelLowering.cpp.

                                                          {
   MaskValue = 0;
   unsigned NumElems = BuildVector->getNumOperands();
   // There are 2 lanes if (NumElems > 8), and 1 lane otherwise.
   unsigned NumLanes = (NumElems - 1) / 8 + 1;
   unsigned NumElemsInLane = NumElems / NumLanes;
 
   // Blend for v16i16 should be symetric for the both lanes.
   for (unsigned i = 0; i < NumElemsInLane; ++i) {
     SDValue EltCond = BuildVector->getOperand(i);
     SDValue SndLaneEltCond =
         (NumLanes == 2) ? BuildVector->getOperand(i + NumElemsInLane) : EltCond;
 
     int Lane1Cond = -1, Lane2Cond = -1;
     if (isa<ConstantSDNode>(EltCond))
       Lane1Cond = !isZero(EltCond);
     if (isa<ConstantSDNode>(SndLaneEltCond))
       Lane2Cond = !isZero(SndLaneEltCond);
 
     if (Lane1Cond == Lane2Cond || Lane2Cond < 0)
       // Lane1Cond != 0, means we want the first argument.
       // Lane1Cond == 0, means we want the second argument.
       // The encoding of this argument is 0 for the first argument, 1
       // for the second. Therefore, invert the condition.
       MaskValue |= !Lane1Cond << i;
     else if (Lane1Cond < 0)
       MaskValue |= !Lane2Cond << i;
     else
       return false;
   }
   return true;
 }

static SDValue buildFromShuffleMostly	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 5913 of file X86ISelLowering.cpp.

                                                                      {
   MVT VT = Op.getSimpleValueType();
 
   // Skip if insert_vec_elt is not supported.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
     return SDValue();
 
   SDLoc DL(Op);
   unsigned NumElems = Op.getNumOperands();
 
   SDValue VecIn1;
   SDValue VecIn2;
   SmallVector<unsigned, 4> InsertIndices;
   SmallVector<int, 8> Mask(NumElems, -1);
 
   for (unsigned i = 0; i != NumElems; ++i) {
     unsigned Opc = Op.getOperand(i).getOpcode();
 
     if (Opc == ISD::UNDEF)
       continue;
 
     if (Opc != ISD::EXTRACT_VECTOR_ELT) {
       // Quit if more than 1 elements need inserting.
       if (InsertIndices.size() > 1)
         return SDValue();
 
       InsertIndices.push_back(i);
       continue;
     }
 
     SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
     SDValue ExtIdx = Op.getOperand(i).getOperand(1);
     // Quit if non-constant index.
     if (!isa<ConstantSDNode>(ExtIdx))
       return SDValue();
     int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx);
 
     // Quit if extracted from vector of different type.
     if (ExtractedFromVec.getValueType() != VT)
       return SDValue();
 
     if (!VecIn1.getNode())
       VecIn1 = ExtractedFromVec;
     else if (VecIn1 != ExtractedFromVec) {
       if (!VecIn2.getNode())
         VecIn2 = ExtractedFromVec;
       else if (VecIn2 != ExtractedFromVec)
         // Quit if more than 2 vectors to shuffle
         return SDValue();
     }
 
     if (ExtractedFromVec == VecIn1)
       Mask[i] = Idx;
     else if (ExtractedFromVec == VecIn2)
       Mask[i] = Idx + NumElems;
   }
 
   if (!VecIn1.getNode())
     return SDValue();
 
   VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
   SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, &Mask[0]);
   for (unsigned i = 0, e = InsertIndices.size(); i != e; ++i) {
     unsigned Idx = InsertIndices[i];
     NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
                      DAG.getIntPtrConstant(Idx));
   }
 
   return NV;
 }

static StructReturnType callIsStructReturn ( const SmallVectorImpl< ISD::OutputArg > & Outs )

static

Definition at line 2128 of file X86ISelLowering.cpp.

                                                               {
   if (Outs.empty())
     return NotStructReturn;
 
   const ISD::ArgFlagsTy &Flags = Outs[0].Flags;
   if (!Flags.isSRet())
     return NotStructReturn;
   if (Flags.isInReg())
     return RegStructReturn;
   return StackStructReturn;
 }

static bool CanFoldXORWithAllOnes ( const SDNode * N )

static

CanFoldXORWithAllOnes - Test whether the XOR operand is a AllOnes vector so it can be folded inside ANDNP.

Definition at line 20479 of file X86ISelLowering.cpp.

                                                    {
   EVT VT = N->getValueType(0);
 
   // Match direct AllOnes for 128 and 256-bit vectors
   if (ISD::isBuildVectorAllOnes(N))
     return true;
 
   // Look through a bit convert.
   if (N->getOpcode() == ISD::BITCAST)
     N = N->getOperand(0).getNode();
 
   // Sometimes the operand may come from a insert_subvector building a 256-bit
   // allones vector
   if (VT.is256BitVector() &&
       N->getOpcode() == ISD::INSERT_SUBVECTOR) {
     SDValue V1 = N->getOperand(0);
     SDValue V2 = N->getOperand(1);
 
     if (V1.getOpcode() == ISD::INSERT_SUBVECTOR &&
         V1.getOperand(0).getOpcode() == ISD::UNDEF &&
         ISD::isBuildVectorAllOnes(V1.getOperand(1).getNode()) &&
         ISD::isBuildVectorAllOnes(V2.getNode()))
       return true;
   }
 
   return false;
 }

static SDValue ChangeVSETULTtoVSETULE	(	SDLoc	dl,
		SDValue	Op1,
		SelectionDAG &	DAG
	)

static

Try to turn a VSETULT into a VSETULE by modifying its second operand Op1. If non-trivial (for example because it's not constant) return an empty value.

Definition at line 12143 of file X86ISelLowering.cpp.

 {
   BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op1.getNode());
   if (!BV)
     return SDValue();
 
   MVT VT = Op1.getSimpleValueType();
   MVT EVT = VT.getVectorElementType();
   unsigned n = VT.getVectorNumElements();
   SmallVector<SDValue, 8> ULTOp1;
 
   for (unsigned i = 0; i < n; ++i) {
     ConstantSDNode *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
     if (!Elt || Elt->isOpaque() || Elt->getValueType(0) != EVT)
       return SDValue();
 
     // Avoid underflow.
     APInt Val = Elt->getAPIntValue();
     if (Val == 0)
       return SDValue();
 
     ULTOp1.push_back(DAG.getConstant(Val - 1, EVT));
   }
 
   return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, ULTOp1);
 }

static bool checkAndUpdateEFLAGSKill	(	MachineBasicBlock::iterator	SelectItr,
		MachineBasicBlock *	BB,
		const TargetRegisterInfo *	TRI
	)

static

Definition at line 17422 of file X86ISelLowering.cpp.

                                                                     {
   // Scan forward through BB for a use/def of EFLAGS.
   MachineBasicBlock::iterator miI(std::next(SelectItr));
   for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
     const MachineInstr& mi = *miI;
     if (mi.readsRegister(X86::EFLAGS))
       return false;
     if (mi.definesRegister(X86::EFLAGS))
       break; // Should have kill-flag - update below.
   }
 
   // If we hit the end of the block, check whether EFLAGS is live into a
   // successor.
   if (miI == BB->end()) {
     for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(),
                                           sEnd = BB->succ_end();
          sItr != sEnd; ++sItr) {
       MachineBasicBlock* succ = *sItr;
       if (succ->isLiveIn(X86::EFLAGS))
         return false;
     }
   }
 
   // We found a def, or hit the end of the basic block and EFLAGS wasn't live
   // out. SelectMI should have a kill flag on EFLAGS.
   SelectItr->addRegisterKilled(X86::EFLAGS, TRI);
   return true;
 }

static SDValue checkBoolTestSetCCCombine	(	SDValue	Cmp,
		X86::CondCode &	CC
	)

static

Definition at line 19832 of file X86ISelLowering.cpp.

                                                                        {
   // Quit if not CMP and SUB with its value result used.
   if (Cmp.getOpcode() != X86ISD::CMP &&
       (Cmp.getOpcode() != X86ISD::SUB || Cmp.getNode()->hasAnyUseOfValue(0)))
       return SDValue();
 
   // Quit if not used as a boolean value.
   if (CC != X86::COND_E && CC != X86::COND_NE)
     return SDValue();
 
   // Check CMP operands. One of them should be 0 or 1 and the other should be
   // an SetCC or extended from it.
   SDValue Op1 = Cmp.getOperand(0);
   SDValue Op2 = Cmp.getOperand(1);
 
   SDValue SetCC;
   const ConstantSDNode* C = nullptr;
   bool needOppositeCond = (CC == X86::COND_E);
   bool checkAgainstTrue = false; // Is it a comparison against 1?
 
   if ((C = dyn_cast<ConstantSDNode>(Op1)))
     SetCC = Op2;
   else if ((C = dyn_cast<ConstantSDNode>(Op2)))
     SetCC = Op1;
   else // Quit if all operands are not constants.
     return SDValue();
 
   if (C->getZExtValue() == 1) {
     needOppositeCond = !needOppositeCond;
     checkAgainstTrue = true;
   } else if (C->getZExtValue() != 0)
     // Quit if the constant is neither 0 or 1.
     return SDValue();
 
   bool truncatedToBoolWithAnd = false;
   // Skip (zext $x), (trunc $x), or (and $x, 1) node.
   while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
          SetCC.getOpcode() == ISD::TRUNCATE ||
          SetCC.getOpcode() == ISD::AND) {
     if (SetCC.getOpcode() == ISD::AND) {
       int OpIdx = -1;
       ConstantSDNode *CS;
       if ((CS = dyn_cast<ConstantSDNode>(SetCC.getOperand(0))) &&
           CS->getZExtValue() == 1)
         OpIdx = 1;
       if ((CS = dyn_cast<ConstantSDNode>(SetCC.getOperand(1))) &&
           CS->getZExtValue() == 1)
         OpIdx = 0;
       if (OpIdx == -1)
         break;
       SetCC = SetCC.getOperand(OpIdx);
       truncatedToBoolWithAnd = true;
     } else
       SetCC = SetCC.getOperand(0);
   }
 
   switch (SetCC.getOpcode()) {
   case X86ISD::SETCC_CARRY:
     // Since SETCC_CARRY gives output based on R = CF ? ~0 : 0, it's unsafe to
     // simplify it if the result of SETCC_CARRY is not canonicalized to 0 or 1,
     // i.e. it's a comparison against true but the result of SETCC_CARRY is not
     // truncated to i1 using 'and'.
     if (checkAgainstTrue && !truncatedToBoolWithAnd)
       break;
     assert(X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B &&
            "Invalid use of SETCC_CARRY!");
     // FALL THROUGH
   case X86ISD::SETCC:
     // Set the condition code or opposite one if necessary.
     CC = X86::CondCode(SetCC.getConstantOperandVal(0));
     if (needOppositeCond)
       CC = X86::GetOppositeBranchCondition(CC);
     return SetCC.getOperand(1);
   case X86ISD::CMOV: {
     // Check whether false/true value has canonical one, i.e. 0 or 1.
     ConstantSDNode *FVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(0));
     ConstantSDNode *TVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(1));
     // Quit if true value is not a constant.
     if (!TVal)
       return SDValue();
     // Quit if false value is not a constant.
     if (!FVal) {
       SDValue Op = SetCC.getOperand(0);
       // Skip 'zext' or 'trunc' node.
       if (Op.getOpcode() == ISD::ZERO_EXTEND ||
           Op.getOpcode() == ISD::TRUNCATE)
         Op = Op.getOperand(0);
       // A special case for rdrand/rdseed, where 0 is set if false cond is
       // found.
       if ((Op.getOpcode() != X86ISD::RDRAND &&
            Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
         return SDValue();
     }
     // Quit if false value is not the constant 0 or 1.
     bool FValIsFalse = true;
     if (FVal && FVal->getZExtValue() != 0) {
       if (FVal->getZExtValue() != 1)
         return SDValue();
       // If FVal is 1, opposite cond is needed.
       needOppositeCond = !needOppositeCond;
       FValIsFalse = false;
     }
     // Quit if TVal is not the constant opposite of FVal.
     if (FValIsFalse && TVal->getZExtValue() != 1)
       return SDValue();
     if (!FValIsFalse && TVal->getZExtValue() != 0)
       return SDValue();
     CC = X86::CondCode(SetCC.getConstantOperandVal(2));
     if (needOppositeCond)
       CC = X86::GetOppositeBranchCondition(CC);
     return SetCC.getOperand(3);
   }
   }
 
   return SDValue();
 }

static bool clobbersFlagRegisters ( const SmallVector< StringRef, 4 > & AsmPieces )

static

Definition at line 22253 of file X86ISelLowering.cpp.

                                                                               {
 
   if (AsmPieces.size() == 3 || AsmPieces.size() == 4) {
     if (std::count(AsmPieces.begin(), AsmPieces.end(), "~{cc}") &&
         std::count(AsmPieces.begin(), AsmPieces.end(), "~{flags}") &&
         std::count(AsmPieces.begin(), AsmPieces.end(), "~{fpsr}")) {
 
       if (AsmPieces.size() == 3)
         return true;
       else if (std::count(AsmPieces.begin(), AsmPieces.end(), "~{dirflag}"))
         return true;
     }
   }
   return false;
 }

static SDValue CMPEQCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 20379 of file X86ISelLowering.cpp.

                                                            {
   unsigned opcode;
 
   // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but
   // we're requiring SSE2 for both.
   if (Subtarget->hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
     SDValue N0 = N->getOperand(0);
     SDValue N1 = N->getOperand(1);
     SDValue CMP0 = N0->getOperand(1);
     SDValue CMP1 = N1->getOperand(1);
     SDLoc DL(N);
 
     // The SETCCs should both refer to the same CMP.
     if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1)
       return SDValue();
 
     SDValue CMP00 = CMP0->getOperand(0);
     SDValue CMP01 = CMP0->getOperand(1);
     EVT     VT    = CMP00.getValueType();
 
     if (VT == MVT::f32 || VT == MVT::f64) {
       bool ExpectingFlags = false;
       // Check for any users that want flags:
       for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
            !ExpectingFlags && UI != UE; ++UI)
         switch (UI->getOpcode()) {
         default:
         case ISD::BR_CC:
         case ISD::BRCOND:
         case ISD::SELECT:
           ExpectingFlags = true;
           break;
         case ISD::CopyToReg:
         case ISD::SIGN_EXTEND:
         case ISD::ZERO_EXTEND:
         case ISD::ANY_EXTEND:
           break;
         }
 
       if (!ExpectingFlags) {
         enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
         enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
 
         if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
           X86::CondCode tmp = cc0;
           cc0 = cc1;
           cc1 = tmp;
         }
 
         if ((cc0 == X86::COND_E  && cc1 == X86::COND_NP) ||
             (cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
           // FIXME: need symbolic constants for these magic numbers.
           // See X86ATTInstPrinter.cpp:printSSECC().
           unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
           if (Subtarget->hasAVX512()) {
             SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00,
                                          CMP01, DAG.getConstant(x86cc, MVT::i8));
             if (N->getValueType(0) != MVT::i1)
               return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0),
                                  FSetCC);
             return FSetCC;
           }
           SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL,
                                               CMP00.getValueType(), CMP00, CMP01,
                                               DAG.getConstant(x86cc, MVT::i8));
 
           bool is64BitFP = (CMP00.getValueType() == MVT::f64);
           MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
 
           if (is64BitFP && !Subtarget->is64Bit()) {
             // On a 32-bit target, we cannot bitcast the 64-bit float to a
             // 64-bit integer, since that's not a legal type. Since
             // OnesOrZeroesF is all ones of all zeroes, we don't need all the
             // bits, but can do this little dance to extract the lowest 32 bits
             // and work with those going forward.
             SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
                                            OnesOrZeroesF);
             SDValue Vector32 = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32,
                                            Vector64);
             OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
                                         Vector32, DAG.getIntPtrConstant(0));
             IntVT = MVT::i32;
           }
 
           SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, IntVT, OnesOrZeroesF);
           SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
                                       DAG.getConstant(1, IntVT));
           SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed);
           return OneBitOfTruth;
         }
       }
     }
   }
   return SDValue();
 }

static bool combineRedundantDWordShuffle	(	SDValue	N,
		MutableArrayRef< int >	Mask,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI
	)

static

Search for a combinable shuffle across a chain ending in pshufd.

We walk up the chain and look for a combinable shuffle, skipping over shuffles that we could hoist this shuffle's transformation past without altering anything.

Definition at line 18496 of file X86ISelLowering.cpp.

                                                                              {
   assert(N.getOpcode() == X86ISD::PSHUFD &&
          "Called with something other than an x86 128-bit half shuffle!");
   SDLoc DL(N);
 
   // Walk up a single-use chain looking for a combinable shuffle.
   SDValue V = N.getOperand(0);
   for (; V.hasOneUse(); V = V.getOperand(0)) {
     switch (V.getOpcode()) {
     default:
       return false; // Nothing combined!
 
     case ISD::BITCAST:
       // Skip bitcasts as we always know the type for the target specific
       // instructions.
       continue;
 
     case X86ISD::PSHUFD:
       // Found another dword shuffle.
       break;
 
     case X86ISD::PSHUFLW:
       // Check that the low words (being shuffled) are the identity in the
       // dword shuffle, and the high words are self-contained.
       if (Mask[0] != 0 || Mask[1] != 1 ||
           !(Mask[2] >= 2 && Mask[2] < 4 && Mask[3] >= 2 && Mask[3] < 4))
         return false;
 
       continue;
 
     case X86ISD::PSHUFHW:
       // Check that the high words (being shuffled) are the identity in the
       // dword shuffle, and the low words are self-contained.
       if (Mask[2] != 2 || Mask[3] != 3 ||
           !(Mask[0] >= 0 && Mask[0] < 2 && Mask[1] >= 0 && Mask[1] < 2))
         return false;
 
       continue;
 
     case X86ISD::UNPCKL:
     case X86ISD::UNPCKH:
       // For either i8 -> i16 or i16 -> i32 unpacks, we can combine a dword
       // shuffle into a preceding word shuffle.
       if (V.getValueType() != MVT::v16i8 && V.getValueType() != MVT::v8i16)
         return false;
 
       // Search for a half-shuffle which we can combine with.
       unsigned CombineOp =
           V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
       if (V.getOperand(0) != V.getOperand(1) ||
           !V->isOnlyUserOf(V.getOperand(0).getNode()))
         return false;
       V = V.getOperand(0);
       do {
         switch (V.getOpcode()) {
         default:
           return false; // Nothing to combine.
 
         case X86ISD::PSHUFLW:
         case X86ISD::PSHUFHW:
           if (V.getOpcode() == CombineOp)
             break;
 
           // Fallthrough!
         case ISD::BITCAST:
           V = V.getOperand(0);
           continue;
         }
         break;
       } while (V.hasOneUse());
       break;
     }
     // Break out of the loop if we break out of the switch.
     break;
   }
 
   if (!V.hasOneUse())
     // We fell out of the loop without finding a viable combining instruction.
     return false;
 
   // Record the old value to use in RAUW-ing.
   SDValue Old = V;
 
   // Merge this node's mask and our incoming mask.
   SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
   for (int &M : Mask)
     M = VMask[M];
   V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
                   getV4X86ShuffleImm8ForMask(Mask, DAG));
 
   // It is possible that one of the combinable shuffles was completely absorbed
   // by the other, just replace it and revisit all users in that case.
   if (Old.getNode() == V.getNode()) {
     DCI.CombineTo(N.getNode(), N.getOperand(0), /*AddTo=*/true);
     return true;
   }
 
   // Replace N with its operand as we're going to combine that shuffle away.
   DAG.ReplaceAllUsesWith(N, N.getOperand(0));
 
   // Replace the combinable shuffle with the combined one, updating all users
   // so that we re-evaluate the chain here.
   DCI.CombineTo(Old.getNode(), V, /*AddTo*/ true);
   return true;
 }

static bool combineRedundantHalfShuffle	(	SDValue	N,
		MutableArrayRef< int >	Mask,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI
	)

static

Search for a combinable shuffle across a chain ending in pshuflw or pshufhw.

We walk up the chain, skipping shuffles of the other half and looking through shuffles which switch halves trying to find a shuffle of the same pair of dwords.

Definition at line 18609 of file X86ISelLowering.cpp.

                                                                             {
   assert(
       (N.getOpcode() == X86ISD::PSHUFLW || N.getOpcode() == X86ISD::PSHUFHW) &&
       "Called with something other than an x86 128-bit half shuffle!");
   SDLoc DL(N);
   unsigned CombineOpcode = N.getOpcode();
 
   // Walk up a single-use chain looking for a combinable shuffle.
   SDValue V = N.getOperand(0);
   for (; V.hasOneUse(); V = V.getOperand(0)) {
     switch (V.getOpcode()) {
     default:
       return false; // Nothing combined!
 
     case ISD::BITCAST:
       // Skip bitcasts as we always know the type for the target specific
       // instructions.
       continue;
 
     case X86ISD::PSHUFLW:
     case X86ISD::PSHUFHW:
       if (V.getOpcode() == CombineOpcode)
         break;
 
       // Other-half shuffles are no-ops.
       continue;
 
     case X86ISD::PSHUFD: {
       // We can only handle pshufd if the half we are combining either stays in
       // its half, or switches to the other half. Bail if one of these isn't
       // true.
       SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
       int DOffset = CombineOpcode == X86ISD::PSHUFLW ? 0 : 2;
       if (!((VMask[DOffset + 0] < 2 && VMask[DOffset + 1] < 2) ||
             (VMask[DOffset + 0] >= 2 && VMask[DOffset + 1] >= 2)))
         return false;
 
       // Map the mask through the pshufd and keep walking up the chain.
       for (int i = 0; i < 4; ++i)
         Mask[i] = 2 * (VMask[DOffset + Mask[i] / 2] % 2) + Mask[i] % 2;
 
       // Switch halves if the pshufd does.
       CombineOpcode =
           VMask[DOffset + Mask[0] / 2] < 2 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
       continue;
     }
     }
     // Break out of the loop if we break out of the switch.
     break;
   }
 
   if (!V.hasOneUse())
     // We fell out of the loop without finding a viable combining instruction.
     return false;
 
   // Record the old value to use in RAUW-ing.
   SDValue Old = V;
 
   // Merge this node's mask and our incoming mask (adjusted to account for all
   // the pshufd instructions encountered).
   SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
   for (int &M : Mask)
     M = VMask[M];
   V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0),
                   getV4X86ShuffleImm8ForMask(Mask, DAG));
 
   // Replace N with its operand as we're going to combine that shuffle away.
   DAG.ReplaceAllUsesWith(N, N.getOperand(0));
 
   // Replace the combinable shuffle with the combined one, updating all users
   // so that we re-evaluate the chain here.
   DCI.CombineTo(Old.getNode(), V, /*AddTo*/ true);
   return true;
 }

static void CommuteVectorShuffleMask	(	SmallVectorImpl< int > &	Mask,
		unsigned	NumElems
	)

static

CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming the two vector operands have swapped position.

Definition at line 3849 of file X86ISelLowering.cpp.

                                                         {
   for (unsigned i = 0; i != NumElems; ++i) {
     int idx = Mask[i];
     if (idx < 0)
       continue;
     else if (idx < (int)NumElems)
       Mask[i] = idx + NumElems;
     else
       Mask[i] = idx - NumElems;
   }
 }

static SDValue Compact8x32ShuffleNode	(	ShuffleVectorSDNode *	SVOp,
		SelectionDAG &	DAG
	)

static

Definition at line 4038 of file X86ISelLowering.cpp.

                                                   {
   MVT VT = SVOp->getSimpleValueType(0);
   SDLoc dl(SVOp);
 
   if (VT != MVT::v8i32 && VT != MVT::v8f32)
     return SDValue();
 
   ArrayRef<int> Mask = SVOp->getMask();
 
   // These are the special masks that may be optimized.
   static const int MaskToOptimizeEven[] = {0, 8, 2, 10, 4, 12, 6, 14};
   static const int MaskToOptimizeOdd[]  = {1, 9, 3, 11, 5, 13, 7, 15};
   bool MatchEvenMask = true;
   bool MatchOddMask  = true;
   for (int i=0; i<8; ++i) {
     if (!isUndefOrEqual(Mask[i], MaskToOptimizeEven[i]))
       MatchEvenMask = false;
     if (!isUndefOrEqual(Mask[i], MaskToOptimizeOdd[i]))
       MatchOddMask = false;
   }
 
   if (!MatchEvenMask && !MatchOddMask)
     return SDValue();
 
   SDValue UndefNode = DAG.getNode(ISD::UNDEF, dl, VT);
 
   SDValue Op0 = SVOp->getOperand(0);
   SDValue Op1 = SVOp->getOperand(1);
 
   if (MatchEvenMask) {
     // Shift the second operand right to 32 bits.
     static const int ShiftRightMask[] = {-1, 0, -1, 2, -1, 4, -1, 6 };
     Op1 = DAG.getVectorShuffle(VT, dl, Op1, UndefNode, ShiftRightMask);
   } else {
     // Shift the first operand left to 32 bits.
     static const int ShiftLeftMask[] = {1, -1, 3, -1, 5, -1, 7, -1 };
     Op0 = DAG.getVectorShuffle(VT, dl, Op0, UndefNode, ShiftLeftMask);
   }
   static const int BlendMask[] = {0, 9, 2, 11, 4, 13, 6, 15};
   return DAG.getVectorShuffle(VT, dl, Op0, Op1, BlendMask);
 }

static SDValue Concat128BitVectors	(	SDValue	V1,
		SDValue	V2,
		EVT	VT,
		unsigned	NumElems,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions. This is used because creating CONCAT_VECTOR nodes of BUILD_VECTORS returns a larger BUILD_VECTOR while we're trying to lower large BUILD_VECTORS.

Definition at line 180 of file X86ISelLowering.cpp.

                                              {
   SDValue V = Insert128BitVector(DAG.getUNDEF(VT), V1, 0, DAG, dl);
   return Insert128BitVector(V, V2, NumElems/2, DAG, dl);
 }

static SDValue Concat256BitVectors	(	SDValue	V1,
		SDValue	V2,
		EVT	VT,
		unsigned	NumElems,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

Definition at line 187 of file X86ISelLowering.cpp.

                                              {
   SDValue V = Insert256BitVector(DAG.getUNDEF(VT), V1, 0, DAG, dl);
   return Insert256BitVector(V, V2, NumElems/2, DAG, dl);
 }

static SDValue CreateCopyOfByValArgument	(	SDValue	Src,
		SDValue	Dst,
		SDValue	Chain,
		ISD::ArgFlagsTy	Flags,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

CreateCopyOfByValArgument - Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute. The copy will be passed as a byval function parameter.

Definition at line 2160 of file X86ISelLowering.cpp.

                                     {
   SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
 
   return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
                        /*isVolatile*/false, /*AlwaysInline=*/true,
                        MachinePointerInfo(), MachinePointerInfo());
 }

static TargetLoweringObjectFile* createTLOF ( const Triple & TT )

static

Definition at line 194 of file X86ISelLowering.cpp.

                                                               {
   if (TT.isOSBinFormatMachO()) {
     if (TT.getArch() == Triple::x86_64)
       return new X86_64MachoTargetObjectFile();
     return new TargetLoweringObjectFileMachO();
   }
 
   if (TT.isOSLinux())
     return new X86LinuxTargetObjectFile();
   if (TT.isOSBinFormatELF())
     return new TargetLoweringObjectFileELF();
   if (TT.isKnownWindowsMSVCEnvironment())
     return new X86WindowsTargetObjectFile();
   if (TT.isOSBinFormatCOFF())
     return new TargetLoweringObjectFileCOFF();
   llvm_unreachable("unknown subtarget type");
 }

static SDValue EltsFromConsecutiveLoads	(	EVT	VT,
		SmallVectorImpl< SDValue > &	Elts,
		SDLoc &	DL,
		SelectionDAG &	DAG,
		bool	isAfterLegalize
	)

static

EltsFromConsecutiveLoads - Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector whose operands are 'elts'.

Example: <load i32 *a, load i32 *a+4, undef, undef> -> zextload a

FIXME: we'd also like to handle the case where the last elements are zero rather than undef via VZEXT_LOAD, but we do not detect that case today. There's even a handy isZeroNode for that purpose.

Definition at line 5643 of file X86ISelLowering.cpp.

                                                               {
   EVT EltVT = VT.getVectorElementType();
   unsigned NumElems = Elts.size();
 
   LoadSDNode *LDBase = nullptr;
   unsigned LastLoadedElt = -1U;
 
   // For each element in the initializer, see if we've found a load or an undef.
   // If we don't find an initial load element, or later load elements are
   // non-consecutive, bail out.
   for (unsigned i = 0; i < NumElems; ++i) {
     SDValue Elt = Elts[i];
 
     if (!Elt.getNode() ||
         (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode())))
       return SDValue();
     if (!LDBase) {
       if (Elt.getNode()->getOpcode() == ISD::UNDEF)
         return SDValue();
       LDBase = cast<LoadSDNode>(Elt.getNode());
       LastLoadedElt = i;
       continue;
     }
     if (Elt.getOpcode() == ISD::UNDEF)
       continue;
 
     LoadSDNode *LD = cast<LoadSDNode>(Elt);
     if (!DAG.isConsecutiveLoad(LD, LDBase, EltVT.getSizeInBits()/8, i))
       return SDValue();
     LastLoadedElt = i;
   }
 
   // If we have found an entire vector of loads and undefs, then return a large
   // load of the entire vector width starting at the base pointer.  If we found
   // consecutive loads for the low half, generate a vzext_load node.
   if (LastLoadedElt == NumElems - 1) {
 
     if (isAfterLegalize &&
         !DAG.getTargetLoweringInfo().isOperationLegal(ISD::LOAD, VT))
       return SDValue();
 
     SDValue NewLd = SDValue();
 
     if (DAG.InferPtrAlignment(LDBase->getBasePtr()) >= 16)
       NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
                           LDBase->getPointerInfo(),
                           LDBase->isVolatile(), LDBase->isNonTemporal(),
                           LDBase->isInvariant(), 0);
     NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
                         LDBase->getPointerInfo(),
                         LDBase->isVolatile(), LDBase->isNonTemporal(),
                         LDBase->isInvariant(), LDBase->getAlignment());
 
     if (LDBase->hasAnyUseOfValue(1)) {
       SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                                      SDValue(LDBase, 1),
                                      SDValue(NewLd.getNode(), 1));
       DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
       DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
                              SDValue(NewLd.getNode(), 1));
     }
 
     return NewLd;
   }
   if (NumElems == 4 && LastLoadedElt == 1 &&
       DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) {
     SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
     SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
     SDValue ResNode =
         DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64,
                                 LDBase->getPointerInfo(),
                                 LDBase->getAlignment(),
                                 false/*isVolatile*/, true/*ReadMem*/,
                                 false/*WriteMem*/);
 
     // Make sure the newly-created LOAD is in the same position as LDBase in
     // terms of dependency. We create a TokenFactor for LDBase and ResNode, and
     // update uses of LDBase's output chain to use the TokenFactor.
     if (LDBase->hasAnyUseOfValue(1)) {
       SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                              SDValue(LDBase, 1), SDValue(ResNode.getNode(), 1));
       DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
       DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
                              SDValue(ResNode.getNode(), 1));
     }
 
     return DAG.getNode(ISD::BITCAST, DL, VT, ResNode);
   }
   return SDValue();
 }

static MachineBasicBlock* EmitMonitor	(	MachineInstr *	MI,
		MachineBasicBlock *	BB,
		const TargetInstrInfo *	TII,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 17059 of file X86ISelLowering.cpp.

                                                                       {
   DebugLoc dl = MI->getDebugLoc();
 
   // Address into RAX/EAX, other two args into ECX, EDX.
   unsigned MemOpc = Subtarget->is64Bit() ? X86::LEA64r : X86::LEA32r;
   unsigned MemReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
   MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(MemOpc), MemReg);
   for (int i = 0; i < X86::AddrNumOperands; ++i)
     MIB.addOperand(MI->getOperand(i));
 
   unsigned ValOps = X86::AddrNumOperands;
   BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::ECX)
     .addReg(MI->getOperand(ValOps).getReg());
   BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::EDX)
     .addReg(MI->getOperand(ValOps+1).getReg());
 
   // The instruction doesn't actually take any operands though.
   BuildMI(*BB, MI, dl, TII->get(X86::MONITORrrr));
 
   MI->eraseFromParent(); // The pseudo is gone now.
   return BB;
 }

static MachineBasicBlock* EmitPCMPSTRI	(	MachineInstr *	MI,
		MachineBasicBlock *	BB,
		const TargetInstrInfo *	TII
	)

static

Definition at line 17024 of file X86ISelLowering.cpp.

                                                                    {
   unsigned Opc;
   switch (MI->getOpcode()) {
   default: llvm_unreachable("illegal opcode!");
   case X86::PCMPISTRIREG:  Opc = X86::PCMPISTRIrr;  break;
   case X86::VPCMPISTRIREG: Opc = X86::VPCMPISTRIrr; break;
   case X86::PCMPISTRIMEM:  Opc = X86::PCMPISTRIrm;  break;
   case X86::VPCMPISTRIMEM: Opc = X86::VPCMPISTRIrm; break;
   case X86::PCMPESTRIREG:  Opc = X86::PCMPESTRIrr;  break;
   case X86::VPCMPESTRIREG: Opc = X86::VPCMPESTRIrr; break;
   case X86::PCMPESTRIMEM:  Opc = X86::PCMPESTRIrm;  break;
   case X86::VPCMPESTRIMEM: Opc = X86::VPCMPESTRIrm; break;
   }
 
   DebugLoc dl = MI->getDebugLoc();
   MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc));
 
   unsigned NumArgs = MI->getNumOperands(); // remove the results
   for (unsigned i = 1; i < NumArgs; ++i) {
     MachineOperand &Op = MI->getOperand(i);
     if (!(Op.isReg() && Op.isImplicit()))
       MIB.addOperand(Op);
   }
   if (MI->hasOneMemOperand())
     MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
 
   BuildMI(*BB, MI, dl,
     TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
     .addReg(X86::ECX);
 
   MI->eraseFromParent();
   return BB;
 }

static MachineBasicBlock* EmitPCMPSTRM	(	MachineInstr *	MI,
		MachineBasicBlock *	BB,
		const TargetInstrInfo *	TII
	)

static

Definition at line 16987 of file X86ISelLowering.cpp.

                                                                    {
   unsigned Opc;
   switch (MI->getOpcode()) {
   default: llvm_unreachable("illegal opcode!");
   case X86::PCMPISTRM128REG:  Opc = X86::PCMPISTRM128rr;  break;
   case X86::VPCMPISTRM128REG: Opc = X86::VPCMPISTRM128rr; break;
   case X86::PCMPISTRM128MEM:  Opc = X86::PCMPISTRM128rm;  break;
   case X86::VPCMPISTRM128MEM: Opc = X86::VPCMPISTRM128rm; break;
   case X86::PCMPESTRM128REG:  Opc = X86::PCMPESTRM128rr;  break;
   case X86::VPCMPESTRM128REG: Opc = X86::VPCMPESTRM128rr; break;
   case X86::PCMPESTRM128MEM:  Opc = X86::PCMPESTRM128rm;  break;
   case X86::VPCMPESTRM128MEM: Opc = X86::VPCMPESTRM128rm; break;
   }
 
   DebugLoc dl = MI->getDebugLoc();
   MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc));
 
   unsigned NumArgs = MI->getNumOperands();
   for (unsigned i = 1; i < NumArgs; ++i) {
     MachineOperand &Op = MI->getOperand(i);
     if (!(Op.isReg() && Op.isImplicit()))
       MIB.addOperand(Op);
   }
   if (MI->hasOneMemOperand())
     MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
 
   BuildMI(*BB, MI, dl,
     TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
     .addReg(X86::XMM0);
 
   MI->eraseFromParent();
   return BB;
 }

static SDValue EmitTailCallStoreRetAddr	(	SelectionDAG &	DAG,
		MachineFunction &	MF,
		SDValue	Chain,
		SDValue	RetAddrFrIdx,
		EVT	PtrVT,
		unsigned	SlotSize,
		int	FPDiff,
		SDLoc	dl
	)

static

EmitTailCallStoreRetAddr - Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0).

Definition at line 2566 of file X86ISelLowering.cpp.

                                                               {
   // Store the return address to the appropriate stack slot.
   if (!FPDiff) return Chain;
   // Calculate the new stack slot for the return address.
   int NewReturnAddrFI =
     MF.getFrameInfo()->CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize,
                                          false);
   SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
   Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
                        MachinePointerInfo::getFixedStack(NewReturnAddrFI),
                        false, false, 0);
   return Chain;
 }

static MachineBasicBlock* EmitXBegin	(	MachineInstr *	MI,
		MachineBasicBlock *	MBB,
		const TargetInstrInfo *	TII
	)

static

Utility function to emit xbegin specifying the start of an RTM region.

Definition at line 16929 of file X86ISelLowering.cpp.

                                                                  {
   DebugLoc DL = MI->getDebugLoc();
 
   const BasicBlock *BB = MBB->getBasicBlock();
   MachineFunction::iterator I = MBB;
   ++I;
 
   // For the v = xbegin(), we generate
   //
   // thisMBB:
   //  xbegin sinkMBB
   //
   // mainMBB:
   //  eax = -1
   //
   // sinkMBB:
   //  v = eax
 
   MachineBasicBlock *thisMBB = MBB;
   MachineFunction *MF = MBB->getParent();
   MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
   MF->insert(I, mainMBB);
   MF->insert(I, sinkMBB);
 
   // Transfer the remainder of BB and its successor edges to sinkMBB.
   sinkMBB->splice(sinkMBB->begin(), MBB,
                   std::next(MachineBasicBlock::iterator(MI)), MBB->end());
   sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
 
   // thisMBB:
   //  xbegin sinkMBB
   //  # fallthrough to mainMBB
   //  # abortion to sinkMBB
   BuildMI(thisMBB, DL, TII->get(X86::XBEGIN_4)).addMBB(sinkMBB);
   thisMBB->addSuccessor(mainMBB);
   thisMBB->addSuccessor(sinkMBB);
 
   // mainMBB:
   //  EAX = -1
   BuildMI(mainMBB, DL, TII->get(X86::MOV32ri), X86::EAX).addImm(-1);
   mainMBB->addSuccessor(sinkMBB);
 
   // sinkMBB:
   // EAX is live into the sinkMBB
   sinkMBB->addLiveIn(X86::EAX);
   BuildMI(*sinkMBB, sinkMBB->begin(), DL,
           TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
     .addReg(X86::EAX);
 
   MI->eraseFromParent();
   return sinkMBB;
 }

static SDValue ExpandHorizontalBinOp	(	const SDValue &	V0,
		const SDValue &	V1,
		SDLoc	DL,
		SelectionDAG &	DAG,
		unsigned	X86Opcode,
		bool	Mode,
		bool	isUndefLO,
		bool	isUndefHI
	)

static

Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.

This is a helper function of PerformBUILD_VECTORCombine. This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.

The kind of horizontal binary operation is defined by X86Opcode.

Mode specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI

Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI

If isUndefLO is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI is set, then UNDEF is propagated to the upper 128-bits of the result.

Definition at line 6187 of file X86ISelLowering.cpp.

                                                                      {
   EVT VT = V0.getValueType();
   assert(VT.is256BitVector() && VT == V1.getValueType() &&
          "Invalid nodes in input!");
 
   unsigned NumElts = VT.getVectorNumElements();
   SDValue V0_LO = Extract128BitVector(V0, 0, DAG, DL);
   SDValue V0_HI = Extract128BitVector(V0, NumElts/2, DAG, DL);
   SDValue V1_LO = Extract128BitVector(V1, 0, DAG, DL);
   SDValue V1_HI = Extract128BitVector(V1, NumElts/2, DAG, DL);
   EVT NewVT = V0_LO.getValueType();
 
   SDValue LO = DAG.getUNDEF(NewVT);
   SDValue HI = DAG.getUNDEF(NewVT);
 
   if (Mode) {
     // Don't emit a horizontal binop if the result is expected to be UNDEF.
     if (!isUndefLO && V0->getOpcode() != ISD::UNDEF)
       LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
     if (!isUndefHI && V1->getOpcode() != ISD::UNDEF)
       HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
   } else {
     // Don't emit a horizontal binop if the result is expected to be UNDEF.
     if (!isUndefLO && (V0_LO->getOpcode() != ISD::UNDEF ||
                        V1_LO->getOpcode() != ISD::UNDEF))
       LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
 
     if (!isUndefHI && (V0_HI->getOpcode() != ISD::UNDEF ||
                        V1_HI->getOpcode() != ISD::UNDEF))
       HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
   }
 
   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
 }

static SDValue Extract128BitVector	(	SDValue	Vec,
		unsigned	IdxVal,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

Generate a DAG to grab 128-bits from a vector > 128 bits. This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit bounday. That makes lowering EXTRACT_VECTOR_ELT operations easier.

Definition at line 118 of file X86ISelLowering.cpp.

                                                                 {
   assert((Vec.getValueType().is256BitVector() ||
           Vec.getValueType().is512BitVector()) && "Unexpected vector size!");
   return ExtractSubVector(Vec, IdxVal, DAG, dl, 128);
 }

static SDValue Extract256BitVector	(	SDValue	Vec,
		unsigned	IdxVal,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

Generate a DAG to grab 256-bits from a 512-bit vector.

Definition at line 126 of file X86ISelLowering.cpp.

                                                                 {
   assert(Vec.getValueType().is512BitVector() && "Unexpected vector size!");
   return ExtractSubVector(Vec, IdxVal, DAG, dl, 256);
 }

static SDValue ExtractSubVector	(	SDValue	Vec,
		unsigned	IdxVal,
		SelectionDAG &	DAG,
		SDLoc	dl,
		unsigned	vectorWidth
	)

static

Definition at line 76 of file X86ISelLowering.cpp.

                                                       {
   assert((vectorWidth == 128 || vectorWidth == 256) &&
          "Unsupported vector width");
   EVT VT = Vec.getValueType();
   EVT ElVT = VT.getVectorElementType();
   unsigned Factor = VT.getSizeInBits()/vectorWidth;
   EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
                                   VT.getVectorNumElements()/Factor);
 
   // Extract from UNDEF is UNDEF.
   if (Vec.getOpcode() == ISD::UNDEF)
     return DAG.getUNDEF(ResultVT);
 
   // Extract the relevant vectorWidth bits.  Generate an EXTRACT_SUBVECTOR
   unsigned ElemsPerChunk = vectorWidth / ElVT.getSizeInBits();
 
   // This is the index of the first element of the vectorWidth-bit chunk
   // we want.
   unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits()) / vectorWidth)
                                * ElemsPerChunk);
 
   // If the input is a buildvector just emit a smaller one.
   if (Vec.getOpcode() == ISD::BUILD_VECTOR)
     return DAG.getNode(ISD::BUILD_VECTOR, dl, ResultVT,
                        makeArrayRef(Vec->op_begin()+NormalizedIdxVal,
                                     ElemsPerChunk));
 
   SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
   SDValue Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec,
                                VecIdx);
 
   return Result;
 
 }

static bool FuncIsMadeTailCallSafe	(	CallingConv::ID	CC,
		bool	GuaranteedTailCallOpt
	)

static

FuncIsMadeTailCallSafe - Return true if the function is being made into a tailcall target by changing its ABI.

Definition at line 2197 of file X86ISelLowering.cpp.

                                                                {
   return GuaranteedTailCallOpt && IsTailCallConvention(CC);
 }

static unsigned getExtractVEXTRACTImmediate	(	SDNode *	N,
		unsigned	vecWidth
	)

static

Definition at line 4696 of file X86ISelLowering.cpp.

                                                                           {
   assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width");
   if (!isa<ConstantSDNode>(N->getOperand(1).getNode()))
     llvm_unreachable("Illegal extract subvector for VEXTRACT");
 
   uint64_t Index =
     cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
 
   MVT VecVT = N->getOperand(0).getSimpleValueType();
   MVT ElVT = VecVT.getVectorElementType();
 
   unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits();
   return Index / NumElemsPerChunk;
 }

static SDValue getGatherNode	(	unsigned	Opc,
		SDValue	Op,
		SelectionDAG &	DAG,
		SDValue	Src,
		SDValue	Mask,
		SDValue	Base,
		SDValue	Index,
		SDValue	ScaleOp,
		SDValue	Chain,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 14204 of file X86ISelLowering.cpp.

                                                               {
   SDLoc dl(Op);
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
   assert(C && "Invalid scale type");
   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
   EVT MaskVT = MVT::getVectorVT(MVT::i1,
                              Index.getSimpleValueType().getVectorNumElements());
   SDValue MaskInReg;
   ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
   if (MaskC)
     MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
   else
     MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
   SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
   SDValue Segment = DAG.getRegister(0, MVT::i32);
   if (Src.getOpcode() == ISD::UNDEF)
     Src = getZeroVector(Op.getValueType(), Subtarget, DAG, dl);
   SDValue Ops[] = {Src, MaskInReg, Base, Scale, Index, Disp, Segment, Chain};
   SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
   SDValue RetOps[] = { SDValue(Res, 0), SDValue(Res, 2) };
   return DAG.getMergeValues(RetOps, dl);
 }

static SDValue getINSERTPS	(	ShuffleVectorSDNode *	SVOp,
		SDLoc &	dl,
		SelectionDAG &	DAG
	)

static

Definition at line 9028 of file X86ISelLowering.cpp.

                                               {
   // Generate an insertps instruction when inserting an f32 from memory onto a
   // v4f32 or when copying a member from one v4f32 to another.
   // We also use it for transferring i32 from one register to another,
   // since it simply copies the same bits.
   // If we're transferring an i32 from memory to a specific element in a
   // register, we output a generic DAG that will match the PINSRD
   // instruction.
   MVT VT = SVOp->getSimpleValueType(0);
   MVT EVT = VT.getVectorElementType();
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   auto Mask = SVOp->getMask();
   assert((VT == MVT::v4f32 || VT == MVT::v4i32) &&
          "unsupported vector type for insertps/pinsrd");
 
   auto FromV1Predicate = [](const int &i) { return i < 4 && i > -1; };
   auto FromV2Predicate = [](const int &i) { return i >= 4; };
   int FromV1 = std::count_if(Mask.begin(), Mask.end(), FromV1Predicate);
 
   SDValue From;
   SDValue To;
   unsigned DestIndex;
   if (FromV1 == 1) {
     From = V1;
     To = V2;
     DestIndex = std::find_if(Mask.begin(), Mask.end(), FromV1Predicate) -
                 Mask.begin();
 
     // If we have 1 element from each vector, we have to check if we're
     // changing V1's element's place. If so, we're done. Otherwise, we
     // should assume we're changing V2's element's place and behave
     // accordingly.
     int FromV2 = std::count_if(Mask.begin(), Mask.end(), FromV2Predicate);
     if (FromV1 == FromV2 && DestIndex == Mask[DestIndex] % 4) {
       From = V2;
       To = V1;
       DestIndex =
           std::find_if(Mask.begin(), Mask.end(), FromV2Predicate) - Mask.begin();
     }
   } else {
     assert(std::count_if(Mask.begin(), Mask.end(), FromV2Predicate) == 1 &&
            "More than one element from V1 and from V2, or no elements from one "
            "of the vectors. This case should not have returned true from "
            "isINSERTPSMask");
     From = V2;
     To = V1;
     DestIndex =
         std::find_if(Mask.begin(), Mask.end(), FromV2Predicate) - Mask.begin();
   }
 
   // Get an index into the source vector in the range [0,4) (the mask is
   // in the range [0,8) because it can address V1 and V2)
   unsigned SrcIndex = Mask[DestIndex] % 4;
   if (MayFoldLoad(From)) {
     // Trivial case, when From comes from a load and is only used by the
     // shuffle. Make it use insertps from the vector that we need from that
     // load.
     SDValue NewLoad =
         NarrowVectorLoadToElement(cast<LoadSDNode>(From), SrcIndex, DAG);
     if (!NewLoad.getNode())
       return SDValue();
 
     if (EVT == MVT::f32) {
       // Create this as a scalar to vector to match the instruction pattern.
       SDValue LoadScalarToVector =
           DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, NewLoad);
       SDValue InsertpsMask = DAG.getIntPtrConstant(DestIndex << 4);
       return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, LoadScalarToVector,
                          InsertpsMask);
     } else { // EVT == MVT::i32
       // If we're getting an i32 from memory, use an INSERT_VECTOR_ELT
       // instruction, to match the PINSRD instruction, which loads an i32 to a
       // certain vector element.
       return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, To, NewLoad,
                          DAG.getConstant(DestIndex, MVT::i32));
     }
   }
 
   // Vector-element-to-vector
   SDValue InsertpsMask = DAG.getIntPtrConstant(DestIndex << 4 | SrcIndex << 6);
   return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, From, InsertpsMask);
 }

static unsigned getInsertVINSERTImmediate	(	SDNode *	N,
		unsigned	vecWidth
	)

static

Definition at line 4711 of file X86ISelLowering.cpp.

                                                                         {
   assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width");
   if (!isa<ConstantSDNode>(N->getOperand(2).getNode()))
     llvm_unreachable("Illegal insert subvector for VINSERT");
 
   uint64_t Index =
     cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
 
   MVT VecVT = N->getSimpleValueType(0);
   MVT ElVT = VecVT.getVectorElementType();
 
   unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits();
   return Index / NumElemsPerChunk;
 }

static SDValue getLegalSplat	(	SelectionDAG &	DAG,
		SDValue	V,
		int	EltNo
	)

static

getLegalSplat - Generate a legal splat with supported x86 shuffles

Definition at line 5022 of file X86ISelLowering.cpp.

                                                                       {
   MVT VT = V.getSimpleValueType();
   SDLoc dl(V);
 
   if (VT.is128BitVector()) {
     V = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V);
     int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo };
     V = DAG.getVectorShuffle(MVT::v4f32, dl, V, DAG.getUNDEF(MVT::v4f32),
                              &SplatMask[0]);
   } else if (VT.is256BitVector()) {
     // To use VPERMILPS to splat scalars, the second half of indicies must
     // refer to the higher part, which is a duplication of the lower one,
     // because VPERMILPS can only handle in-lane permutations.
     int SplatMask[8] = { EltNo, EltNo, EltNo, EltNo,
                          EltNo+4, EltNo+4, EltNo+4, EltNo+4 };
 
     V = DAG.getNode(ISD::BITCAST, dl, MVT::v8f32, V);
     V = DAG.getVectorShuffle(MVT::v8f32, dl, V, DAG.getUNDEF(MVT::v8f32),
                              &SplatMask[0]);
   } else
     llvm_unreachable("Vector size not supported");
 
   return DAG.getNode(ISD::BITCAST, dl, VT, V);
 }

static void getMaxByValAlign	(	Type *	Ty,
		unsigned &	MaxAlign
	)

static

getMaxByValAlign - Helper for getByValTypeAlignment to determine the desired ByVal argument alignment.

Definition at line 1646 of file X86ISelLowering.cpp.

                                                            {
   if (MaxAlign == 16)
     return;
   if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
     if (VTy->getBitWidth() == 128)
       MaxAlign = 16;
   } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     unsigned EltAlign = 0;
     getMaxByValAlign(ATy->getElementType(), EltAlign);
     if (EltAlign > MaxAlign)
       MaxAlign = EltAlign;
   } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
       unsigned EltAlign = 0;
       getMaxByValAlign(STy->getElementType(i), EltAlign);
       if (EltAlign > MaxAlign)
         MaxAlign = EltAlign;
       if (MaxAlign == 16)
         break;
     }
   }
 }

static SDValue getMOVDDup	(	SDValue &	Op,
		SDLoc &	dl,
		SDValue	V1,
		SelectionDAG &	DAG
	)

static

Definition at line 8907 of file X86ISelLowering.cpp.

                                                                           {
   MVT VT = Op.getSimpleValueType();
 
   // Canonizalize to v2f64.
   V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1);
   return DAG.getNode(ISD::BITCAST, dl, VT,
                      getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64,
                                           V1, DAG));
 }

static SDValue getMOVHighToLow	(	SDValue &	Op,
		SDLoc &	dl,
		SelectionDAG &	DAG
	)

static

Definition at line 8937 of file X86ISelLowering.cpp.

                                                                    {
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   MVT VT = Op.getSimpleValueType();
 
   assert((VT == MVT::v4i32 || VT == MVT::v4f32) &&
          "unsupported shuffle type");
 
   if (V2.getOpcode() == ISD::UNDEF)
     V2 = V1;
 
   // v4i32 or v4f32
   return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG);
 }

static SDValue getMOVL	(	SelectionDAG &	DAG,
		SDLoc	dl,
		EVT	VT,
		SDValue	V1,
		SDValue	V2
	)

static

getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width.

Definition at line 4966 of file X86ISelLowering.cpp.

                                    {
   unsigned NumElems = VT.getVectorNumElements();
   SmallVector<int, 8> Mask;
   Mask.push_back(NumElems);
   for (unsigned i = 1; i != NumElems; ++i)
     Mask.push_back(i);
   return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
 }

static SDValue getMOVLowToHigh	(	SDValue &	Op,
		SDLoc &	dl,
		SelectionDAG &	DAG,
		bool	HasSSE2
	)

static

Definition at line 8918 of file X86ISelLowering.cpp.

                                       {
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   MVT VT = Op.getSimpleValueType();
 
   assert(VT != MVT::v2i64 && "unsupported shuffle type");
 
   if (HasSSE2 && VT == MVT::v2f64)
     return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG);
 
   // v4f32 or v4i32: canonizalized to v4f32 (which is legal for SSE1)
   return DAG.getNode(ISD::BITCAST, dl, VT,
                      getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32,
                            DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V1),
                            DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V2), DAG));
 }

static SDValue getMOVLP	(	SDValue &	Op,
		SDLoc &	dl,
		SelectionDAG &	DAG,
		bool	HasSSE2
	)

static

Definition at line 8953 of file X86ISelLowering.cpp.

                                                                           {
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   MVT VT = Op.getSimpleValueType();
   unsigned NumElems = VT.getVectorNumElements();
 
   // Use MOVLPS and MOVLPD in case V1 or V2 are loads. During isel, the second
   // operand of these instructions is only memory, so check if there's a
   // potencial load folding here, otherwise use SHUFPS or MOVSD to match the
   // same masks.
   bool CanFoldLoad = false;
 
   // Trivial case, when V2 comes from a load.
   if (MayFoldVectorLoad(V2))
     CanFoldLoad = true;
 
   // When V1 is a load, it can be folded later into a store in isel, example:
   //  (store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), addr:$src1)
   //    turns into:
   //  (MOVLPSmr addr:$src1, VR128:$src2)
   // So, recognize this potential and also use MOVLPS or MOVLPD
   else if (MayFoldVectorLoad(V1) && MayFoldIntoStore(Op))
     CanFoldLoad = true;
 
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   if (CanFoldLoad) {
     if (HasSSE2 && NumElems == 2)
       return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG);
 
     if (NumElems == 4)
       // If we don't care about the second element, proceed to use movss.
       if (SVOp->getMaskElt(1) != -1)
         return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG);
   }
 
   // movl and movlp will both match v2i64, but v2i64 is never matched by
   // movl earlier because we make it strict to avoid messing with the movlp load
   // folding logic (see the code above getMOVLP call). Match it here then,
   // this is horrible, but will stay like this until we move all shuffle
   // matching to x86 specific nodes. Note that for the 1st condition all
   // types are matched with movsd.
   if (HasSSE2) {
     // FIXME: isMOVLMask should be checked and matched before getMOVLP,
     // as to remove this logic from here, as much as possible
     if (NumElems == 2 || !isMOVLMask(SVOp->getMask(), VT))
       return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
     return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
   }
 
   assert(VT != MVT::v4i32 && "unsupported shuffle type");
 
   // Invert the operand order and use SHUFPS to match it.
   return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V2, V1,
                               getShuffleSHUFImmediate(SVOp), DAG);
 }

static unsigned getNumOfConsecutiveZeros	(	ShuffleVectorSDNode *	SVOp,
		unsigned	NumElems,
		bool	ZerosFromLeft,
		SelectionDAG &	DAG,
		unsigned	PreferredNum = `-1U`
	)

static

getNumOfConsecutiveZeros - Return the number of elements of a vector shuffle operation which come from a consecutively from a zero. The search can start in two different directions, from left or right. We count undefs as zeros until PreferredNum is reached.

Definition at line 5254 of file X86ISelLowering.cpp.

                                                                       {
   unsigned NumZeros = 0;
   for (unsigned i = 0; i != NumElems; ++i) {
     unsigned Index = ZerosFromLeft ? i : NumElems - i - 1;
     SDValue Elt = getShuffleScalarElt(SVOp, Index, DAG, 0);
     if (!Elt.getNode())
       break;
 
     if (X86::isZeroNode(Elt))
       ++NumZeros;
     else if (Elt.getOpcode() == ISD::UNDEF) // Undef as zero up to PreferredNum.
       NumZeros = std::min(NumZeros + 1, PreferredNum);
     else
       break;
   }
 
   return NumZeros;
 }

static SDValue getOnesVector	(	MVT	VT,
		bool	HasInt256,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

getOnesVector - Returns a vector of specified type with all bits set. Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately. Then bitcast to their original type, ensuring they get CSE'd.

Definition at line 4932 of file X86ISelLowering.cpp.

                                        {
   assert(VT.isVector() && "Expected a vector type");
 
   SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32);
   SDValue Vec;
   if (VT.is256BitVector()) {
     if (HasInt256) { // AVX2
       SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops);
     } else { // AVX
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
       Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl);
     }
   } else if (VT.is128BitVector()) {
     Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
   } else
     llvm_unreachable("Unexpected vector type");
 
   return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
 }

static SDValue getPrefetchNode	(	unsigned	Opc,
		SDValue	Op,
		SelectionDAG &	DAG,
		SDValue	Mask,
		SDValue	Base,
		SDValue	Index,
		SDValue	ScaleOp,
		SDValue	Chain
	)

static

Definition at line 14254 of file X86ISelLowering.cpp.

                                                                {
   SDLoc dl(Op);
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
   assert(C && "Invalid scale type");
   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
   SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
   SDValue Segment = DAG.getRegister(0, MVT::i32);
   EVT MaskVT =
     MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
   SDValue MaskInReg;
   ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
   if (MaskC)
     MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
   else
     MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
   //SDVTList VTs = DAG.getVTList(MVT::Other);
   SDValue Ops[] = {MaskInReg, Base, Scale, Index, Disp, Segment, Chain};
   SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
   return SDValue(Res, 0);
 }

static SDValue getPSHUFB	(	ArrayRef< int >	MaskVals,
		SDValue	V1,
		SDLoc &	dl,
		SelectionDAG &	DAG
	)

static

Generate a PSHUFB if possible. Selects elements from V1 according to MaskVals. MaskVals[OutputIdx] = InputIdx specifies that we want to shuffle the element at InputIdx in V1 to OutputIdx in the result. If MaskVals refers to elements outside of V1 or is undef (-1), insert a zero.

Definition at line 8097 of file X86ISelLowering.cpp.

                                             {
   MVT VT = V1.getSimpleValueType();
   assert(VT.is128BitVector() || VT.is256BitVector());
 
   MVT EltVT = VT.getVectorElementType();
   unsigned EltSizeInBytes = EltVT.getSizeInBits() / 8;
   unsigned NumElts = VT.getVectorNumElements();
 
   SmallVector<SDValue, 32> PshufbMask;
   for (unsigned OutputIdx = 0; OutputIdx < NumElts; ++OutputIdx) {
     int InputIdx = MaskVals[OutputIdx];
     unsigned InputByteIdx;
 
     if (InputIdx < 0 || NumElts <= (unsigned)InputIdx)
       InputByteIdx = 0x80;
     else {
       // Cross lane is not allowed.
       if (ShuffleCrosses128bitLane(VT, InputIdx, OutputIdx))
         return SDValue();
       InputByteIdx = InputIdx * EltSizeInBytes;
       // Index is an byte offset within the 128-bit lane.
       InputByteIdx &= 0xf;
     }
 
     for (unsigned j = 0; j < EltSizeInBytes; ++j) {
       PshufbMask.push_back(DAG.getConstant(InputByteIdx, MVT::i8));
       if (InputByteIdx != 0x80)
         ++InputByteIdx;
     }
   }
 
   MVT ShufVT = MVT::getVectorVT(MVT::i8, PshufbMask.size());
   if (ShufVT != VT)
     V1 = DAG.getNode(ISD::BITCAST, dl, ShufVT, V1);
   return DAG.getNode(X86ISD::PSHUFB, dl, ShufVT, V1,
                      DAG.getNode(ISD::BUILD_VECTOR, dl, ShufVT, PshufbMask));
 }

static SmallVector<int, 4> getPSHUFShuffleMask ( SDValue N )

static

Get the PSHUF-style mask from PSHUF node.

This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.

Definition at line 18468 of file X86ISelLowering.cpp.

                                                           {
   SmallVector<int, 4> Mask;
   bool IsUnary;
   bool HaveMask = getTargetShuffleMask(N.getNode(), N.getSimpleValueType(), Mask, IsUnary);
   (void)HaveMask;
   assert(HaveMask);
 
   switch (N.getOpcode()) {
   case X86ISD::PSHUFD:
     return Mask;
   case X86ISD::PSHUFLW:
     Mask.resize(4);
     return Mask;
   case X86ISD::PSHUFHW:
     Mask.erase(Mask.begin(), Mask.begin() + 4);
     for (int &M : Mask)
       M -= 4;
     return Mask;
   default:
     llvm_unreachable("No valid shuffle instruction found!");
   }
 }

static void getReadPerformanceCounter	(	SDNode *	N,
		SDLoc	DL,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget,
		SmallVectorImpl< SDValue > &	Results
	)

static

Definition at line 14279 of file X86ISelLowering.cpp.

                                                                  {
   assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
   SDValue LO, HI;
 
   // The ECX register is used to select the index of the performance counter
   // to read.
   SDValue Chain = DAG.getCopyToReg(N->getOperand(0), DL, X86::ECX,
                                    N->getOperand(2));
   SDValue rd = DAG.getNode(X86ISD::RDPMC_DAG, DL, Tys, Chain);
 
   // Reads the content of a 64-bit performance counter and returns it in the
   // registers EDX:EAX.
   if (Subtarget->is64Bit()) {
     LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1));
     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
                             LO.getValue(2));
   } else {
     LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1));
     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
                             LO.getValue(2));
   }
   Chain = HI.getValue(1);
 
   if (Subtarget->is64Bit()) {
     // The EAX register is loaded with the low-order 32 bits. The EDX register
     // is loaded with the supported high-order bits of the counter.
     SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
                               DAG.getConstant(32, MVT::i8));
     Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
     Results.push_back(Chain);
     return;
   }
 
   // Use a buildpair to merge the two 32-bit values into a 64-bit one.
   SDValue Ops[] = { LO, HI };
   SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
   Results.push_back(Pair);
   Results.push_back(Chain);
 }

static void getReadTimeStampCounter	(	SDNode *	N,
		SDLoc	DL,
		unsigned	Opcode,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget,
		SmallVectorImpl< SDValue > &	Results
	)

static

Definition at line 14325 of file X86ISelLowering.cpp.

                                                                  {
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
   SDValue rd = DAG.getNode(Opcode, DL, Tys, N->getOperand(0));
   SDValue LO, HI;
 
   // The processor's time-stamp counter (a 64-bit MSR) is stored into the
   // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
   // and the EAX register is loaded with the low-order 32 bits.
   if (Subtarget->is64Bit()) {
     LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1));
     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
                             LO.getValue(2));
   } else {
     LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1));
     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
                             LO.getValue(2));
   }
   SDValue Chain = HI.getValue(1);
 
   if (Opcode == X86ISD::RDTSCP_DAG) {
     assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
 
     // Instruction RDTSCP loads the IA32:TSC_AUX_MSR (address C000_0103H) into
     // the ECX register. Add 'ecx' explicitly to the chain.
     SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32,
                                      HI.getValue(2));
     // Explicitly store the content of ECX at the location passed in input
     // to the 'rdtscp' intrinsic.
     Chain = DAG.getStore(ecx.getValue(1), DL, ecx, N->getOperand(2),
                          MachinePointerInfo(), false, false, 0);
   }
 
   if (Subtarget->is64Bit()) {
     // The EDX register is loaded with the high-order 32 bits of the MSR, and
     // the EAX register is loaded with the low-order 32 bits.
     SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
                               DAG.getConstant(32, MVT::i8));
     Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
     Results.push_back(Chain);
     return;
   }
 
   // Use a buildpair to merge the two 32-bit values into a 64-bit one.
   SDValue Ops[] = { LO, HI };
   SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
   Results.push_back(Pair);
   Results.push_back(Chain);
 }

static SDValue getScatterNode	(	unsigned	Opc,
		SDValue	Op,
		SelectionDAG &	DAG,
		SDValue	Src,
		SDValue	Mask,
		SDValue	Base,
		SDValue	Index,
		SDValue	ScaleOp,
		SDValue	Chain
	)

static

Definition at line 14231 of file X86ISelLowering.cpp.

                                                                               {
   SDLoc dl(Op);
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
   assert(C && "Invalid scale type");
   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
   SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
   SDValue Segment = DAG.getRegister(0, MVT::i32);
   EVT MaskVT = MVT::getVectorVT(MVT::i1,
                              Index.getSimpleValueType().getVectorNumElements());
   SDValue MaskInReg;
   ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
   if (MaskC)
     MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
   else
     MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
   SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other);
   SDValue Ops[] = {Base, Scale, Index, Disp, Segment, MaskInReg, Src, Chain};
   SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
   return SDValue(Res, 1);
 }

static unsigned getShufflePALIGNRImmediate ( ShuffleVectorSDNode * SVOp )

static

getShufflePALIGNRImmediate - Return the appropriate immediate to shuffle the specified VECTOR_SHUFFLE mask with the PALIGNR instruction.

Definition at line 4673 of file X86ISelLowering.cpp.

                                                                       {
   MVT VT = SVOp->getSimpleValueType(0);
   unsigned EltSize = VT.is512BitVector() ? 1 :
     VT.getVectorElementType().getSizeInBits() >> 3;
 
   unsigned NumElts = VT.getVectorNumElements();
   unsigned NumLanes = VT.is512BitVector() ? 1 : VT.getSizeInBits()/128;
   unsigned NumLaneElts = NumElts/NumLanes;
 
   int Val = 0;
   unsigned i;
   for (i = 0; i != NumElts; ++i) {
     Val = SVOp->getMaskElt(i);
     if (Val >= 0)
       break;
   }
   if (Val >= (int)NumElts)
     Val -= NumElts - NumLaneElts;
 
   assert(Val - i > 0 && "PALIGNR imm should be positive");
   return (Val - i) * EltSize;
 }

static unsigned getShufflePSHUFHWImmediate ( ShuffleVectorSDNode * N )

static

getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle the specified VECTOR_SHUFFLE mask with the PSHUFHW instruction.

Definition at line 4625 of file X86ISelLowering.cpp.

                                                                    {
   MVT VT = N->getSimpleValueType(0);
 
   assert((VT == MVT::v8i16 || VT == MVT::v16i16) &&
          "Unsupported vector type for PSHUFHW");
 
   unsigned NumElts = VT.getVectorNumElements();
 
   unsigned Mask = 0;
   for (unsigned l = 0; l != NumElts; l += 8) {
     // 8 nodes per lane, but we only care about the last 4.
     for (unsigned i = 0; i < 4; ++i) {
       int Elt = N->getMaskElt(l+i+4);
       if (Elt < 0) continue;
       Elt &= 0x3; // only 2-bits.
       Mask |= Elt << (i * 2);
     }
   }
 
   return Mask;
 }

static unsigned getShufflePSHUFLWImmediate ( ShuffleVectorSDNode * N )

static

getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle the specified VECTOR_SHUFFLE mask with the PSHUFLW instruction.

Definition at line 4649 of file X86ISelLowering.cpp.

                                                                    {
   MVT VT = N->getSimpleValueType(0);
 
   assert((VT == MVT::v8i16 || VT == MVT::v16i16) &&
          "Unsupported vector type for PSHUFHW");
 
   unsigned NumElts = VT.getVectorNumElements();
 
   unsigned Mask = 0;
   for (unsigned l = 0; l != NumElts; l += 8) {
     // 8 nodes per lane, but we only care about the first 4.
     for (unsigned i = 0; i < 4; ++i) {
       int Elt = N->getMaskElt(l+i);
       if (Elt < 0) continue;
       Elt &= 0x3; // only 2-bits
       Mask |= Elt << (i * 2);
     }
   }
 
   return Mask;
 }

static SDValue getShuffleScalarElt	(	SDNode *	N,
		unsigned	Index,
		SelectionDAG &	DAG,
		unsigned	Depth
	)

static

getShuffleScalarElt - Returns the scalar element that will make up the ith element of the result of the vector shuffle.

Definition at line 5188 of file X86ISelLowering.cpp.

                                                    {
   if (Depth == 6)
     return SDValue();  // Limit search depth.
 
   SDValue V = SDValue(N, 0);
   EVT VT = V.getValueType();
   unsigned Opcode = V.getOpcode();
 
   // Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
   if (const ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) {
     int Elt = SV->getMaskElt(Index);
 
     if (Elt < 0)
       return DAG.getUNDEF(VT.getVectorElementType());
 
     unsigned NumElems = VT.getVectorNumElements();
     SDValue NewV = (Elt < (int)NumElems) ? SV->getOperand(0)
                                          : SV->getOperand(1);
     return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth+1);
   }
 
   // Recurse into target specific vector shuffles to find scalars.
   if (isTargetShuffle(Opcode)) {
     MVT ShufVT = V.getSimpleValueType();
     unsigned NumElems = ShufVT.getVectorNumElements();
     SmallVector<int, 16> ShuffleMask;
     bool IsUnary;
 
     if (!getTargetShuffleMask(N, ShufVT, ShuffleMask, IsUnary))
       return SDValue();
 
     int Elt = ShuffleMask[Index];
     if (Elt < 0)
       return DAG.getUNDEF(ShufVT.getVectorElementType());
 
     SDValue NewV = (Elt < (int)NumElems) ? N->getOperand(0)
                                          : N->getOperand(1);
     return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG,
                                Depth+1);
   }
 
   // Actual nodes that may contain scalar elements
   if (Opcode == ISD::BITCAST) {
     V = V.getOperand(0);
     EVT SrcVT = V.getValueType();
     unsigned NumElems = VT.getVectorNumElements();
 
     if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems)
       return SDValue();
   }
 
   if (V.getOpcode() == ISD::SCALAR_TO_VECTOR)
     return (Index == 0) ? V.getOperand(0)
                         : DAG.getUNDEF(VT.getVectorElementType());
 
   if (V.getOpcode() == ISD::BUILD_VECTOR)
     return V.getOperand(Index);
 
   return SDValue();
 }

static unsigned getShuffleSHUFImmediate ( ShuffleVectorSDNode * N )

static

getShuffleSHUFImmediate - Return the appropriate immediate to shuffle the specified VECTOR_SHUFFLE mask with PSHUF* and SHUFP* instructions. Handles 128-bit and 256-bit.

Definition at line 4595 of file X86ISelLowering.cpp.

                                                                 {
   MVT VT = N->getSimpleValueType(0);
 
   assert((VT.getSizeInBits() >= 128) &&
          "Unsupported vector type for PSHUF/SHUFP");
 
   // Handle 128 and 256-bit vector lengths. AVX defines PSHUF/SHUFP to operate
   // independently on 128-bit lanes.
   unsigned NumElts = VT.getVectorNumElements();
   unsigned NumLanes = VT.getSizeInBits()/128;
   unsigned NumLaneElts = NumElts/NumLanes;
 
   assert((NumLaneElts == 2 || NumLaneElts == 4 || NumLaneElts == 8) &&
          "Only supports 2, 4 or 8 elements per lane");
 
   unsigned Shift = (NumLaneElts >= 4) ? 1 : 0;
   unsigned Mask = 0;
   for (unsigned i = 0; i != NumElts; ++i) {
     int Elt = N->getMaskElt(i);
     if (Elt < 0) continue;
     Elt &= NumLaneElts - 1;
     unsigned ShAmt = (i << Shift) % 8;
     Mask |= Elt << ShAmt;
   }
 
   return Mask;
 }

static SDValue getShuffleVectorZeroOrUndef	(	SDValue	V2,
		unsigned	Idx,
		bool	IsZero,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified vector of zero or undef vector. This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).

Definition at line 5090 of file X86ISelLowering.cpp.

                                                               {
   MVT VT = V2.getSimpleValueType();
   SDValue V1 = IsZero
     ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
   unsigned NumElems = VT.getVectorNumElements();
   SmallVector<int, 16> MaskVec;
   for (unsigned i = 0; i != NumElems; ++i)
     // If this is the insertion idx, put the low elt of V2 here.
     MaskVec.push_back(i == Idx ? NumElems : i);
   return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, &MaskVec[0]);
 }

static unsigned getShuffleVPERM2X128Immediate ( ShuffleVectorSDNode * SVOp )

static

getShuffleVPERM2X128Immediate - Return the appropriate immediate to shuffle the specified VECTOR_MASK mask with VPERM2F128/VPERM2I128 instructions.

Definition at line 4334 of file X86ISelLowering.cpp.

                                                                          {
   MVT VT = SVOp->getSimpleValueType(0);
 
   unsigned HalfSize = VT.getVectorNumElements()/2;
 
   unsigned FstHalf = 0, SndHalf = 0;
   for (unsigned i = 0; i < HalfSize; ++i) {
     if (SVOp->getMaskElt(i) > 0) {
       FstHalf = SVOp->getMaskElt(i)/HalfSize;
       break;
     }
   }
   for (unsigned i = HalfSize; i < HalfSize*2; ++i) {
     if (SVOp->getMaskElt(i) > 0) {
       SndHalf = SVOp->getMaskElt(i)/HalfSize;
       break;
     }
   }
 
   return (FstHalf | (SndHalf << 4));
 }

static bool getTargetShuffleMask	(	SDNode *	N,
		MVT	VT,
		SmallVectorImpl< int > &	Mask,
		bool &	IsUnary
	)

static

getTargetShuffleMask - Calculates the shuffle mask corresponding to the target specific opcode. Returns true if the Mask could be calculated. Sets IsUnary to true if only uses one source.

Definition at line 5108 of file X86ISelLowering.cpp.

                                                                             {
   unsigned NumElems = VT.getVectorNumElements();
   SDValue ImmN;
 
   IsUnary = false;
   switch(N->getOpcode()) {
   case X86ISD::SHUFP:
     ImmN = N->getOperand(N->getNumOperands()-1);
     DecodeSHUFPMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
     break;
   case X86ISD::UNPCKH:
     DecodeUNPCKHMask(VT, Mask);
     break;
   case X86ISD::UNPCKL:
     DecodeUNPCKLMask(VT, Mask);
     break;
   case X86ISD::MOVHLPS:
     DecodeMOVHLPSMask(NumElems, Mask);
     break;
   case X86ISD::MOVLHPS:
     DecodeMOVLHPSMask(NumElems, Mask);
     break;
   case X86ISD::PALIGNR:
     ImmN = N->getOperand(N->getNumOperands()-1);
     DecodePALIGNRMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
     break;
   case X86ISD::PSHUFD:
   case X86ISD::VPERMILP:
     ImmN = N->getOperand(N->getNumOperands()-1);
     DecodePSHUFMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
     IsUnary = true;
     break;
   case X86ISD::PSHUFHW:
     ImmN = N->getOperand(N->getNumOperands()-1);
     DecodePSHUFHWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
     IsUnary = true;
     break;
   case X86ISD::PSHUFLW:
     ImmN = N->getOperand(N->getNumOperands()-1);
     DecodePSHUFLWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
     IsUnary = true;
     break;
   case X86ISD::VPERMI:
     ImmN = N->getOperand(N->getNumOperands()-1);
     DecodeVPERMMask(cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
     IsUnary = true;
     break;
   case X86ISD::MOVSS:
   case X86ISD::MOVSD: {
     // The index 0 always comes from the first element of the second source,
     // this is why MOVSS and MOVSD are used in the first place. The other
     // elements come from the other positions of the first source vector
     Mask.push_back(NumElems);
     for (unsigned i = 1; i != NumElems; ++i) {
       Mask.push_back(i);
     }
     break;
   }
   case X86ISD::VPERM2X128:
     ImmN = N->getOperand(N->getNumOperands()-1);
     DecodeVPERM2X128Mask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
     if (Mask.empty()) return false;
     break;
   case X86ISD::MOVDDUP:
   case X86ISD::MOVLHPD:
   case X86ISD::MOVLPD:
   case X86ISD::MOVLPS:
   case X86ISD::MOVSHDUP:
   case X86ISD::MOVSLDUP:
     // Not yet implemented
     return false;
   default: llvm_unreachable("unknown target shuffle node");
   }
 
   return true;
 }

static SDValue getTargetShuffleNode	(	unsigned	Opc,
		SDLoc	dl,
		EVT	VT,
		SDValue	V1,
		SelectionDAG &	DAG
	)

static

Definition at line 3405 of file X86ISelLowering.cpp.

                                                                    {
   switch(Opc) {
   default: llvm_unreachable("Unknown x86 shuffle node");
   case X86ISD::MOVSHDUP:
   case X86ISD::MOVSLDUP:
   case X86ISD::MOVDDUP:
     return DAG.getNode(Opc, dl, VT, V1);
   }
 }

static SDValue getTargetShuffleNode	(	unsigned	Opc,
		SDLoc	dl,
		EVT	VT,
		SDValue	V1,
		unsigned	TargetMask,
		SelectionDAG &	DAG
	)

static

Definition at line 3416 of file X86ISelLowering.cpp.

                                                        {
   switch(Opc) {
   default: llvm_unreachable("Unknown x86 shuffle node");
   case X86ISD::PSHUFD:
   case X86ISD::PSHUFHW:
   case X86ISD::PSHUFLW:
   case X86ISD::VPERMILP:
   case X86ISD::VPERMI:
     return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8));
   }
 }

static SDValue getTargetShuffleNode	(	unsigned	Opc,
		SDLoc	dl,
		EVT	VT,
		SDValue	V1,
		SDValue	V2,
		unsigned	TargetMask,
		SelectionDAG &	DAG
	)

static

Definition at line 3430 of file X86ISelLowering.cpp.

                                                        {
   switch(Opc) {
   default: llvm_unreachable("Unknown x86 shuffle node");
   case X86ISD::PALIGNR:
   case X86ISD::SHUFP:
   case X86ISD::VPERM2X128:
     return DAG.getNode(Opc, dl, VT, V1, V2,
                        DAG.getConstant(TargetMask, MVT::i8));
   }
 }

static SDValue getTargetShuffleNode	(	unsigned	Opc,
		SDLoc	dl,
		EVT	VT,
		SDValue	V1,
		SDValue	V2,
		SelectionDAG &	DAG
	)

static

Definition at line 3443 of file X86ISelLowering.cpp.

                                                                                {
   switch(Opc) {
   default: llvm_unreachable("Unknown x86 shuffle node");
   case X86ISD::MOVLHPS:
   case X86ISD::MOVLHPD:
   case X86ISD::MOVHLPS:
   case X86ISD::MOVLPS:
   case X86ISD::MOVLPD:
   case X86ISD::MOVSS:
   case X86ISD::MOVSD:
   case X86ISD::UNPCKL:
   case X86ISD::UNPCKH:
     return DAG.getNode(Opc, dl, VT, V1, V2);
   }
 }

static SDValue getTargetVShiftByConstNode	(	unsigned	Opc,
		SDLoc	dl,
		MVT	VT,
		SDValue	SrcOp,
		uint64_t	ShiftAmt,
		SelectionDAG &	DAG
	)

static

Definition at line 13375 of file X86ISelLowering.cpp.

                                                              {
   MVT ElementType = VT.getVectorElementType();
 
   // Fold this packed shift into its first operand if ShiftAmt is 0.
   if (ShiftAmt == 0)
     return SrcOp;
 
   // Check for ShiftAmt >= element width
   if (ShiftAmt >= ElementType.getSizeInBits()) {
     if (Opc == X86ISD::VSRAI)
       ShiftAmt = ElementType.getSizeInBits() - 1;
     else
       return DAG.getConstant(0, VT);
   }
 
   assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
          && "Unknown target vector shift-by-constant node");
 
   // Fold this packed vector shift into a build vector if SrcOp is a
   // vector of Constants or UNDEFs, and SrcOp valuetype is the same as VT.
   if (VT == SrcOp.getSimpleValueType() &&
       ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
     SmallVector<SDValue, 8> Elts;
     unsigned NumElts = SrcOp->getNumOperands();
     ConstantSDNode *ND;
 
     switch(Opc) {
     default: llvm_unreachable(nullptr);
     case X86ISD::VSHLI:
       for (unsigned i=0; i!=NumElts; ++i) {
         SDValue CurrentOp = SrcOp->getOperand(i);
         if (CurrentOp->getOpcode() == ISD::UNDEF) {
           Elts.push_back(CurrentOp);
           continue;
         }
         ND = cast<ConstantSDNode>(CurrentOp);
         const APInt &C = ND->getAPIntValue();
         Elts.push_back(DAG.getConstant(C.shl(ShiftAmt), ElementType));
       }
       break;
     case X86ISD::VSRLI:
       for (unsigned i=0; i!=NumElts; ++i) {
         SDValue CurrentOp = SrcOp->getOperand(i);
         if (CurrentOp->getOpcode() == ISD::UNDEF) {
           Elts.push_back(CurrentOp);
           continue;
         }
         ND = cast<ConstantSDNode>(CurrentOp);
         const APInt &C = ND->getAPIntValue();
         Elts.push_back(DAG.getConstant(C.lshr(ShiftAmt), ElementType));
       }
       break;
     case X86ISD::VSRAI:
       for (unsigned i=0; i!=NumElts; ++i) {
         SDValue CurrentOp = SrcOp->getOperand(i);
         if (CurrentOp->getOpcode() == ISD::UNDEF) {
           Elts.push_back(CurrentOp);
           continue;
         }
         ND = cast<ConstantSDNode>(CurrentOp);
         const APInt &C = ND->getAPIntValue();
         Elts.push_back(DAG.getConstant(C.ashr(ShiftAmt), ElementType));
       }
       break;
     }
 
     return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts);
   }
 
   return DAG.getNode(Opc, dl, VT, SrcOp, DAG.getConstant(ShiftAmt, MVT::i8));
 }

static SDValue getTargetVShiftNode	(	unsigned	Opc,
		SDLoc	dl,
		MVT	VT,
		SDValue	SrcOp,
		SDValue	ShAmt,
		SelectionDAG &	DAG
	)

static

Definition at line 13451 of file X86ISelLowering.cpp.

                                                       {
   assert(ShAmt.getValueType() == MVT::i32 && "ShAmt is not i32");
 
   // Catch shift-by-constant.
   if (ConstantSDNode *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
     return getTargetVShiftByConstNode(Opc, dl, VT, SrcOp,
                                       CShAmt->getZExtValue(), DAG);
 
   // Change opcode to non-immediate version
   switch (Opc) {
     default: llvm_unreachable("Unknown target vector shift node");
     case X86ISD::VSHLI: Opc = X86ISD::VSHL; break;
     case X86ISD::VSRLI: Opc = X86ISD::VSRL; break;
     case X86ISD::VSRAI: Opc = X86ISD::VSRA; break;
   }
 
   // Need to build a vector containing shift amount
   // Shift amount is 32-bits, but SSE instructions read 64-bit, so fill with 0
   SDValue ShOps[4];
   ShOps[0] = ShAmt;
   ShOps[1] = DAG.getConstant(0, MVT::i32);
   ShOps[2] = ShOps[3] = DAG.getUNDEF(MVT::i32);
   ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, ShOps);
 
   // The return type has to be a 128-bit type with the same element
   // type as the input type.
   MVT EltVT = VT.getVectorElementType();
   EVT ShVT = MVT::getVectorVT(EltVT, 128/EltVT.getSizeInBits());
 
   ShAmt = DAG.getNode(ISD::BITCAST, dl, ShVT, ShAmt);
   return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
 }

static SDValue GetTLSADDR	(	SelectionDAG &	DAG,
		SDValue	Chain,
		GlobalAddressSDNode *	GA,
		SDValue *	InFlag,
		const EVT	PtrVT,
		unsigned	ReturnReg,
		unsigned char	OperandFlags,
		bool	LocalDynamic = `false`
	)

static

Definition at line 10339 of file X86ISelLowering.cpp.

                                                                   {
   MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
   SDLoc dl(GA);
   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
                                            GA->getValueType(0),
                                            GA->getOffset(),
                                            OperandFlags);
 
   X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
                                            : X86ISD::TLSADDR;
 
   if (InFlag) {
     SDValue Ops[] = { Chain,  TGA, *InFlag };
     Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
   } else {
     SDValue Ops[]  = { Chain, TGA };
     Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
   }
 
   // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
   MFI->setAdjustsStack(true);
 
   SDValue Flag = Chain.getValue(1);
   return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag);
 }

static int getUnderlyingExtractedFromVec	(	SDValue &	ExtractedFromVec,
		SDValue	ExtIdx
	)

static

For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.

Modifies ExtractedFromVec to the real vector and returns the real index.

Definition at line 5883 of file X86ISelLowering.cpp.

                                                          {
   int Idx = cast<ConstantSDNode>(ExtIdx)->getZExtValue();
   if (!isa<ShuffleVectorSDNode>(ExtractedFromVec))
     return Idx;
 
   // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already
   // lowered this:
   //   (extract_vector_elt (v8f32 %vreg1), Constant<6>)
   // to:
   //   (extract_vector_elt (vector_shuffle<2,u,u,u>
   //                           (extract_subvector (v8f32 %vreg0), Constant<4>),
   //                           undef)
   //                       Constant<0>)
   // In this case the vector is the extract_subvector expression and the index
   // is 2, as specified by the shuffle.
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(ExtractedFromVec);
   SDValue ShuffleVec = SVOp->getOperand(0);
   MVT ShuffleVecVT = ShuffleVec.getSimpleValueType();
   assert(ShuffleVecVT.getVectorElementType() ==
          ExtractedFromVec.getSimpleValueType().getVectorElementType());
 
   int ShuffleIdx = SVOp->getMaskElt(Idx);
   if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) {
     ExtractedFromVec = ShuffleVec;
     return ShuffleIdx;
   }
   return Idx;
 }

static SDValue getUnpackh	(	SelectionDAG &	DAG,
		SDLoc	dl,
		MVT	VT,
		SDValue	V1,
		SDValue	V2
	)

static

getUnpackh - Returns a vector_shuffle node for an unpackh operation.

Definition at line 4989 of file X86ISelLowering.cpp.

                                       {
   unsigned NumElems = VT.getVectorNumElements();
   SmallVector<int, 8> Mask;
   for (unsigned i = 0, Half = NumElems/2; i != Half; ++i) {
     Mask.push_back(i + Half);
     Mask.push_back(i + NumElems + Half);
   }
   return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
 }

static SDValue getUnpackl	(	SelectionDAG &	DAG,
		SDLoc	dl,
		MVT	VT,
		SDValue	V1,
		SDValue	V2
	)

static

getUnpackl - Returns a vector_shuffle node for an unpackl operation.

Definition at line 4977 of file X86ISelLowering.cpp.

                                       {
   unsigned NumElems = VT.getVectorNumElements();
   SmallVector<int, 8> Mask;
   for (unsigned i = 0, e = NumElems/2; i != e; ++i) {
     Mask.push_back(i);
     Mask.push_back(i + NumElems);
   }
   return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
 }

static SDValue getV4X86ShuffleImm8ForMask	(	ArrayRef< int >	Mask,
		SelectionDAG &	DAG
	)

static

Get a 4-lane 8-bit shuffle immediate for a mask.

This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.

NB: We rely heavily on "undef" masks preserving the input lane.

Definition at line 6928 of file X86ISelLowering.cpp.

                                                              {
   assert(Mask.size() == 4 && "Only 4-lane shuffle masks");
   assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!");
   assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!");
   assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!");
   assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!");
 
   unsigned Imm = 0;
   Imm |= (Mask[0] == -1 ? 0 : Mask[0]) << 0;
   Imm |= (Mask[1] == -1 ? 1 : Mask[1]) << 2;
   Imm |= (Mask[2] == -1 ? 2 : Mask[2]) << 4;
   Imm |= (Mask[3] == -1 ? 3 : Mask[3]) << 6;
   return DAG.getConstant(Imm, MVT::i8);
 }

static SDValue getVShift	(	bool	isLeft,
		EVT	VT,
		SDValue	SrcOp,
		unsigned	NumBits,
		SelectionDAG &	DAG,
		const TargetLowering &	TLI,
		SDLoc	dl
	)

static

getVShift - Return a vector logical shift node.

Definition at line 5547 of file X86ISelLowering.cpp.

                                                               {
   assert(VT.is128BitVector() && "Unknown type for VShift");
   EVT ShVT = MVT::v2i64;
   unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
   SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp);
   return DAG.getNode(ISD::BITCAST, dl, VT,
                      DAG.getNode(Opc, dl, ShVT, SrcOp,
                              DAG.getConstant(NumBits,
                                   TLI.getScalarShiftAmountTy(SrcOp.getValueType()))));
 }

static SDValue getVZextMovL	(	MVT	VT,
		MVT	OpVT,
		SDValue	SrcOp,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget,
		SDLoc	dl
	)

static

getVZextMovL - Return a zero-extending vector move low node.

Definition at line 8604 of file X86ISelLowering.cpp.

                                                                      {
   if (VT == MVT::v2f64 || VT == MVT::v4f32) {
     LoadSDNode *LD = nullptr;
     if (!isScalarLoadToVector(SrcOp.getNode(), &LD))
       LD = dyn_cast<LoadSDNode>(SrcOp);
     if (!LD) {
       // movssrr and movsdrr do not clear top bits. Try to use movd, movq
       // instead.
       MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32;
       if ((ExtVT != MVT::i64 || Subtarget->is64Bit()) &&
           SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR &&
           SrcOp.getOperand(0).getOpcode() == ISD::BITCAST &&
           SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) {
         // PR2108
         OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32;
         return DAG.getNode(ISD::BITCAST, dl, VT,
                            DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
                                        DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
                                                    OpVT,
                                                    SrcOp.getOperand(0)
                                                           .getOperand(0))));
       }
     }
   }
 
   return DAG.getNode(ISD::BITCAST, dl, VT,
                      DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
                                  DAG.getNode(ISD::BITCAST, dl,
                                              OpVT, SrcOp)));
 }

static SDValue getZeroVector	(	EVT	VT,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

getZeroVector - Returns a vector of specified type with all zero elements.

Definition at line 4885 of file X86ISelLowering.cpp.

                                                           {
   assert(VT.isVector() && "Expected a vector type");
 
   // Always build SSE zero vectors as <4 x i32> bitcasted
   // to their dest type. This ensures they get CSE'd.
   SDValue Vec;
   if (VT.is128BitVector()) {  // SSE
     if (Subtarget->hasSSE2()) {  // SSE2
       SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
     } else { // SSE1
       SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32);
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst);
     }
   } else if (VT.is256BitVector()) { // AVX
     if (Subtarget->hasInt256()) { // AVX2
       SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
       SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops);
     } else {
       // 256-bit logic and arithmetic instructions in AVX are all
       // floating-point, no support for integer ops. Emit fp zeroed vectors.
       SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32);
       SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops);
     }
   } else if (VT.is512BitVector()) { // AVX-512
       SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
       SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst,
                         Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
       Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops);
   } else if (VT.getScalarType() == MVT::i1) {
     assert(VT.getVectorNumElements() <= 16 && "Unexpected vector type");
     SDValue Cst = DAG.getTargetConstant(0, MVT::i1);
     SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
     return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
   } else
     llvm_unreachable("Unexpected vector type");
 
   return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
 }

static bool hasFPCMov ( unsigned X86CC )

static

hasFPCMov - is there a floating point cmov for the specific X86 condition code. Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.

Definition at line 3643 of file X86ISelLowering.cpp.

                                       {
   switch (X86CC) {
   default:
     return false;
   case X86::COND_B:
   case X86::COND_BE:
   case X86::COND_E:
   case X86::COND_P:
   case X86::COND_A:
   case X86::COND_AE:
   case X86::COND_NE:
   case X86::COND_NP:
     return true;
   }
 }

static bool hasNonFlagsUse ( SDValue Op )

static

return true if Op has a use that doesn't just read flags.

Definition at line 11634 of file X86ISelLowering.cpp.

                                        {
   for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE;
        ++UI) {
     SDNode *User = *UI;
     unsigned UOpNo = UI.getOperandNo();
     if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) {
       // Look pass truncate.
       UOpNo = User->use_begin().getOperandNo();
       User = *User->use_begin();
     }
 
     if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC &&
         !(User->getOpcode() == ISD::SELECT && UOpNo == 0))
       return true;
   }
   return false;
 }

static void InitIntinsicsMap ( )

static

Definition at line 14398 of file X86ISelLowering.cpp.

                                {
   static bool Initialized = false;
   if (Initialized) 
     return;
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_qps_512,
                                 IntrinsicData(GATHER, X86::VGATHERQPSZrm, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_qps_512,
                                 IntrinsicData(GATHER, X86::VGATHERQPSZrm, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_qpd_512,
                                 IntrinsicData(GATHER, X86::VGATHERQPDZrm, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_dpd_512,
                                 IntrinsicData(GATHER, X86::VGATHERDPDZrm, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_dps_512,
                                 IntrinsicData(GATHER, X86::VGATHERDPSZrm, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_qpi_512, 
                                 IntrinsicData(GATHER, X86::VPGATHERQDZrm, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_qpq_512, 
                                 IntrinsicData(GATHER, X86::VPGATHERQQZrm, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_dpi_512, 
                                 IntrinsicData(GATHER, X86::VPGATHERDDZrm, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gather_dpq_512, 
                                 IntrinsicData(GATHER, X86::VPGATHERDQZrm, 0)));
 
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatter_qps_512,
                                 IntrinsicData(SCATTER, X86::VSCATTERQPSZmr, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatter_qpd_512, 
                                 IntrinsicData(SCATTER, X86::VSCATTERQPDZmr, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatter_dpd_512, 
                                 IntrinsicData(SCATTER, X86::VSCATTERDPDZmr, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatter_dps_512, 
                                 IntrinsicData(SCATTER, X86::VSCATTERDPSZmr, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatter_qpi_512, 
                                 IntrinsicData(SCATTER, X86::VPSCATTERQDZmr, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatter_qpq_512, 
                                 IntrinsicData(SCATTER, X86::VPSCATTERQQZmr, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatter_dpi_512, 
                                 IntrinsicData(SCATTER, X86::VPSCATTERDDZmr, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatter_dpq_512, 
                                 IntrinsicData(SCATTER, X86::VPSCATTERDQZmr, 0)));
    
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gatherpf_qps_512, 
                                 IntrinsicData(PREFETCH, X86::VGATHERPF0QPSm,
                                                         X86::VGATHERPF1QPSm)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gatherpf_qpd_512, 
                                 IntrinsicData(PREFETCH, X86::VGATHERPF0QPDm,
                                                         X86::VGATHERPF1QPDm)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gatherpf_dpd_512, 
                                 IntrinsicData(PREFETCH, X86::VGATHERPF0DPDm,
                                                         X86::VGATHERPF1DPDm)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_gatherpf_dps_512, 
                                 IntrinsicData(PREFETCH, X86::VGATHERPF0DPSm,
                                                         X86::VGATHERPF1DPSm)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatterpf_qps_512, 
                                 IntrinsicData(PREFETCH, X86::VSCATTERPF0QPSm,
                                                         X86::VSCATTERPF1QPSm)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatterpf_qpd_512, 
                                 IntrinsicData(PREFETCH, X86::VSCATTERPF0QPDm,
                                                         X86::VSCATTERPF1QPDm)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatterpf_dpd_512, 
                                 IntrinsicData(PREFETCH, X86::VSCATTERPF0DPDm,
                                                         X86::VSCATTERPF1DPDm)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_avx512_scatterpf_dps_512, 
                                 IntrinsicData(PREFETCH, X86::VSCATTERPF0DPSm,
                                                         X86::VSCATTERPF1DPSm)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdrand_16,
                                 IntrinsicData(RDRAND, X86ISD::RDRAND, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdrand_32,
                                 IntrinsicData(RDRAND, X86ISD::RDRAND, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdrand_64,
                                 IntrinsicData(RDRAND, X86ISD::RDRAND, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdseed_16,
                                 IntrinsicData(RDSEED, X86ISD::RDSEED, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdseed_32,
                                 IntrinsicData(RDSEED, X86ISD::RDSEED, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdseed_64,
                                 IntrinsicData(RDSEED, X86ISD::RDSEED, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_xtest,
                                 IntrinsicData(XTEST,  X86ISD::XTEST,  0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdtsc,
                                 IntrinsicData(RDTSC,  X86ISD::RDTSC_DAG, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdtscp,
                                 IntrinsicData(RDTSC,  X86ISD::RDTSCP_DAG, 0)));
   IntrMap.insert(std::make_pair(Intrinsic::x86_rdpmc,
                                 IntrinsicData(RDPMC,  X86ISD::RDPMC_DAG, 0)));
   Initialized = true;
 }

static SDValue Insert128BitVector	(	SDValue	Result,
		SDValue	Vec,
		unsigned	IdxVal,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

Generate a DAG to put 128-bits into a vector > 128 bits. This sets things up to match to an AVX VINSERTF128/VINSERTI128 or AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a simple superregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit bounday. That makes lowering INSERT_VECTOR_ELT operations easier.

Definition at line 162 of file X86ISelLowering.cpp.

                                             {
   assert(Vec.getValueType().is128BitVector() && "Unexpected vector size!");
   return InsertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
 }

static SDValue Insert256BitVector	(	SDValue	Result,
		SDValue	Vec,
		unsigned	IdxVal,
		SelectionDAG &	DAG,
		SDLoc	dl
	)

static

Definition at line 169 of file X86ISelLowering.cpp.

                                             {
   assert(Vec.getValueType().is256BitVector() && "Unexpected vector size!");
   return InsertSubVector(Result, Vec, IdxVal, DAG, dl, 256);
 }

static SDValue InsertSubVector	(	SDValue	Result,
		SDValue	Vec,
		unsigned	IdxVal,
		SelectionDAG &	DAG,
		SDLoc	dl,
		unsigned	vectorWidth
	)

static

Definition at line 132 of file X86ISelLowering.cpp.

                                                                {
   assert((vectorWidth == 128 || vectorWidth == 256) &&
          "Unsupported vector width");
   // Inserting UNDEF is Result
   if (Vec.getOpcode() == ISD::UNDEF)
     return Result;
   EVT VT = Vec.getValueType();
   EVT ElVT = VT.getVectorElementType();
   EVT ResultVT = Result.getValueType();
 
   // Insert the relevant vectorWidth bits.
   unsigned ElemsPerChunk = vectorWidth/ElVT.getSizeInBits();
 
   // This is the index of the first element of the vectorWidth-bit chunk
   // we want.
   unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits())/vectorWidth)
                                * ElemsPerChunk);
 
   SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec,
                      VecIdx);
 }

static bool isAllOnes ( SDValue V )

static

Definition at line 11949 of file X86ISelLowering.cpp.

                                  {
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
   return C && C->isAllOnesValue();
 }

static bool isAndOrOfSetCCs	(	SDValue	Op,
		unsigned &	Opc
	)

static

Definition at line 12817 of file X86ISelLowering.cpp.

                                                        {
   Opc = Op.getOpcode();
   if (Opc != ISD::OR && Opc != ISD::AND)
     return false;
   return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
           Op.getOperand(0).hasOneUse() &&
           Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
           Op.getOperand(1).hasOneUse());
 }

static bool isBlendMask	(	ArrayRef< int >	MaskVals,
		MVT	VT,
		bool	hasSSE41,
		bool	hasInt256,
		unsigned *	MaskOut = `nullptr`
	)

static

Definition at line 8009 of file X86ISelLowering.cpp.

                                                                      {
   MVT EltVT = VT.getVectorElementType();
 
   // There is no blend with immediate in AVX-512.
   if (VT.is512BitVector())
     return false;
 
   if (!hasSSE41 || EltVT == MVT::i8)
     return false;
   if (!hasInt256 && VT == MVT::v16i16)
     return false;
 
   unsigned MaskValue = 0;
   unsigned NumElems = VT.getVectorNumElements();
   // There are 2 lanes if (NumElems > 8), and 1 lane otherwise.
   unsigned NumLanes = (NumElems - 1) / 8 + 1;
   unsigned NumElemsInLane = NumElems / NumLanes;
 
   // Blend for v16i16 should be symetric for the both lanes.
   for (unsigned i = 0; i < NumElemsInLane; ++i) {
 
     int SndLaneEltIdx = (NumLanes == 2) ? MaskVals[i + NumElemsInLane] : -1;
     int EltIdx = MaskVals[i];
 
     if ((EltIdx < 0 || EltIdx == (int)i) &&
         (SndLaneEltIdx < 0 || SndLaneEltIdx == (int)(i + NumElemsInLane)))
       continue;
 
     if (((unsigned)EltIdx == (i + NumElems)) &&
         (SndLaneEltIdx < 0 ||
          (unsigned)SndLaneEltIdx == i + NumElems + NumElemsInLane))
       MaskValue |= (1 << i);
     else
       return false;
   }
 
   if (MaskOut)
     *MaskOut = MaskValue;
   return true;
 }

static bool IsCCallConvention ( CallingConv::ID CC )

static

Return true if the calling convention is a C calling convention.

Definition at line 2178 of file X86ISelLowering.cpp.

                                                 {
   return (CC == CallingConv::C || CC == CallingConv::X86_64_Win64 ||
           CC == CallingConv::X86_64_SysV);
 }

static bool isCommutedMOVLMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	V2IsSplat = `false`,
		bool	V2IsUndef = `false`
	)

static

isCommutedMOVLMask - Returns true if the shuffle mask is except the reverse of what x86 movss want. X86 movs requires the lowest element to be lowest element of vector 2 and the other elements to come from vector 1 in order.

Definition at line 4433 of file X86ISelLowering.cpp.

                                                                                {
   if (!VT.is128BitVector())
     return false;
 
   unsigned NumOps = VT.getVectorNumElements();
   if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16)
     return false;
 
   if (!isUndefOrEqual(Mask[0], 0))
     return false;
 
   for (unsigned i = 1; i != NumOps; ++i)
     if (!(isUndefOrEqual(Mask[i], i+NumOps) ||
           (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) ||
           (V2IsSplat && isUndefOrEqual(Mask[i], NumOps))))
       return false;
 
   return true;
 }

static bool isHorizontalBinOp	(	const BuildVectorSDNode *	N,
		unsigned	Opcode,
		SelectionDAG &	DAG,
		unsigned	BaseIdx,
		unsigned	LastIdx,
		SDValue &	V0,
		SDValue &	V1
	)

static

Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1.

This is a helper function of PerformBUILD_VECTORCombine. This function checks that the build_vector N in input implements a horizontal operation. Parameter Opcode defines the kind of horizontal operation to match. For example, if Opcode is equal to ISD::ADD, then this function checks if N implements a horizontal arithmetic add; if instead Opcode is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.

This function only analyzes elements of N whose indices are in range [BaseIdx, LastIdx).

Definition at line 6079 of file X86ISelLowering.cpp.

                                                         {
   EVT VT = N->getValueType(0);
 
   assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!");
   assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&
          "Invalid Vector in input!");
   
   bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD);
   bool CanFold = true;
   unsigned ExpectedVExtractIdx = BaseIdx;
   unsigned NumElts = LastIdx - BaseIdx;
   V0 = DAG.getUNDEF(VT);
   V1 = DAG.getUNDEF(VT);
 
   // Check if N implements a horizontal binop.
   for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) {
     SDValue Op = N->getOperand(i + BaseIdx);
 
     // Skip UNDEFs.
     if (Op->getOpcode() == ISD::UNDEF) {
       // Update the expected vector extract index.
       if (i * 2 == NumElts)
         ExpectedVExtractIdx = BaseIdx;
       ExpectedVExtractIdx += 2;
       continue;
     }
 
     CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
 
     if (!CanFold)
       break;
 
     SDValue Op0 = Op.getOperand(0);
     SDValue Op1 = Op.getOperand(1);
 
     // Try to match the following pattern:
     // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
     CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
         Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
         Op0.getOperand(0) == Op1.getOperand(0) &&
         isa<ConstantSDNode>(Op0.getOperand(1)) &&
         isa<ConstantSDNode>(Op1.getOperand(1)));
     if (!CanFold)
       break;
 
     unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
     unsigned I1 = cast<ConstantSDNode>(Op1.getOperand(1))->getZExtValue();
 
     if (i * 2 < NumElts) {
       if (V0.getOpcode() == ISD::UNDEF)
         V0 = Op0.getOperand(0);
     } else {
       if (V1.getOpcode() == ISD::UNDEF)
         V1 = Op0.getOperand(0);
       if (i * 2 == NumElts)
         ExpectedVExtractIdx = BaseIdx;
     }
 
     SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
     if (I0 == ExpectedVExtractIdx)
       CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
     else if (IsCommutable && I1 == ExpectedVExtractIdx) {
       // Try to match the following dag sequence:
       // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
       CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
     } else
       CanFold = false;
 
     ExpectedVExtractIdx += 2;
   }
 
   return CanFold;
 }

static bool isHorizontalBinOp	(	SDValue &	LHS,
		SDValue &	RHS,
		bool	IsCommutative
	)

static

isHorizontalBinOp - Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'. Note that the binary operation should have the property that if one of the operands is UNDEF then the result is UNDEF.

Definition at line 21305 of file X86ISelLowering.cpp.

                                                                               {
   // Look for the following pattern: if
   //   A = < float a0, float a1, float a2, float a3 >
   //   B = < float b0, float b1, float b2, float b3 >
   // and
   //   LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6>
   //   RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7>
   // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
   // which is A horizontal-op B.
 
   // At least one of the operands should be a vector shuffle.
   if (LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
       RHS.getOpcode() != ISD::VECTOR_SHUFFLE)
     return false;
 
   MVT VT = LHS.getSimpleValueType();
 
   assert((VT.is128BitVector() || VT.is256BitVector()) &&
          "Unsupported vector type for horizontal add/sub");
 
   // Handle 128 and 256-bit vector lengths. AVX defines horizontal add/sub to
   // operate independently on 128-bit lanes.
   unsigned NumElts = VT.getVectorNumElements();
   unsigned NumLanes = VT.getSizeInBits()/128;
   unsigned NumLaneElts = NumElts / NumLanes;
   assert((NumLaneElts % 2 == 0) &&
          "Vector type should have an even number of elements in each lane");
   unsigned HalfLaneElts = NumLaneElts/2;
 
   // View LHS in the form
   //   LHS = VECTOR_SHUFFLE A, B, LMask
   // If LHS is not a shuffle then pretend it is the shuffle
   //   LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1>
   // NOTE: in what follows a default initialized SDValue represents an UNDEF of
   // type VT.
   SDValue A, B;
   SmallVector<int, 16> LMask(NumElts);
   if (LHS.getOpcode() == ISD::VECTOR_SHUFFLE) {
     if (LHS.getOperand(0).getOpcode() != ISD::UNDEF)
       A = LHS.getOperand(0);
     if (LHS.getOperand(1).getOpcode() != ISD::UNDEF)
       B = LHS.getOperand(1);
     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(LHS.getNode())->getMask();
     std::copy(Mask.begin(), Mask.end(), LMask.begin());
   } else {
     if (LHS.getOpcode() != ISD::UNDEF)
       A = LHS;
     for (unsigned i = 0; i != NumElts; ++i)
       LMask[i] = i;
   }
 
   // Likewise, view RHS in the form
   //   RHS = VECTOR_SHUFFLE C, D, RMask
   SDValue C, D;
   SmallVector<int, 16> RMask(NumElts);
   if (RHS.getOpcode() == ISD::VECTOR_SHUFFLE) {
     if (RHS.getOperand(0).getOpcode() != ISD::UNDEF)
       C = RHS.getOperand(0);
     if (RHS.getOperand(1).getOpcode() != ISD::UNDEF)
       D = RHS.getOperand(1);
     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(RHS.getNode())->getMask();
     std::copy(Mask.begin(), Mask.end(), RMask.begin());
   } else {
     if (RHS.getOpcode() != ISD::UNDEF)
       C = RHS;
     for (unsigned i = 0; i != NumElts; ++i)
       RMask[i] = i;
   }
 
   // Check that the shuffles are both shuffling the same vectors.
   if (!(A == C && B == D) && !(A == D && B == C))
     return false;
 
   // If everything is UNDEF then bail out: it would be better to fold to UNDEF.
   if (!A.getNode() && !B.getNode())
     return false;
 
   // If A and B occur in reverse order in RHS, then "swap" them (which means
   // rewriting the mask).
   if (A != C)
     CommuteVectorShuffleMask(RMask, NumElts);
 
   // At this point LHS and RHS are equivalent to
   //   LHS = VECTOR_SHUFFLE A, B, LMask
   //   RHS = VECTOR_SHUFFLE A, B, RMask
   // Check that the masks correspond to performing a horizontal operation.
   for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
     for (unsigned i = 0; i != NumLaneElts; ++i) {
       int LIdx = LMask[i+l], RIdx = RMask[i+l];
 
       // Ignore any UNDEF components.
       if (LIdx < 0 || RIdx < 0 ||
           (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) ||
           (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts)))
         continue;
 
       // Check that successive elements are being operated on.  If not, this is
       // not a horizontal operation.
       unsigned Src = (i/HalfLaneElts); // each lane is split between srcs
       int Index = 2*(i%HalfLaneElts) + NumElts*Src + l;
       if (!(LIdx == Index && RIdx == Index + 1) &&
           !(IsCommutative && LIdx == Index + 1 && RIdx == Index))
         return false;
     }
   }
 
   LHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
   RHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
   return true;
 }

static bool isINSERT64x4Mask	(	ArrayRef< int >	Mask,
		MVT	VT,
		unsigned int *	Imm
	)

static

Definition at line 4255 of file X86ISelLowering.cpp.

                                                                             {
   if (!VT.is512BitVector())
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
   unsigned HalfSize = NumElts/2;
   if (isSequentialOrUndefInRange(Mask, 0, HalfSize, 0)) {
     if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, NumElts)) {
       *Imm = 1;
       return true;
     }
   }
   if (isSequentialOrUndefInRange(Mask, 0, HalfSize, NumElts)) {
     if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, HalfSize)) {
       *Imm = 0;
       return true;
     }
   }
   return false;
 }

static bool isINSERTPSMask	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

isINSERTPSMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to INSERTPS. i. e: If all but one element come from the same vector.

Definition at line 4006 of file X86ISelLowering.cpp.

                                                        {
   // TODO: Deal with AVX's VINSERTPS
   if (!VT.is128BitVector() || (VT != MVT::v4f32 && VT != MVT::v4i32))
     return false;
 
   unsigned CorrectPosV1 = 0;
   unsigned CorrectPosV2 = 0;
   for (int i = 0, e = (int)VT.getVectorNumElements(); i != e; ++i) {
     if (Mask[i] == -1) {
       ++CorrectPosV1;
       ++CorrectPosV2;
       continue;
     }
 
     if (Mask[i] == i)
       ++CorrectPosV1;
     else if (Mask[i] == i + 4)
       ++CorrectPosV2;
   }
 
   if (CorrectPosV1 == 3 || CorrectPosV2 == 3)
     // We have 3 elements (undefs count as elements from any vector) from one
     // vector, and one from another.
     return true;
 
   return false;
 }

static bool isMOVDDUPMask	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to 128-bit version of MOVDDUP.

Definition at line 4525 of file X86ISelLowering.cpp.

                                                       {
   if (!VT.is128BitVector())
     return false;
 
   unsigned e = VT.getVectorNumElements() / 2;
   for (unsigned i = 0; i != e; ++i)
     if (!isUndefOrEqual(Mask[i], i))
       return false;
   for (unsigned i = 0; i != e; ++i)
     if (!isUndefOrEqual(Mask[e+i], i))
       return false;
   return true;
 }

static bool isMOVDDUPYMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	HasFp256
	)

static

isMOVDDUPYMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to 256-bit version of MOVDDUP.

Definition at line 4505 of file X86ISelLowering.cpp.

                                                                       {
   if (!HasFp256 || !VT.is256BitVector())
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
   if (NumElts != 4)
     return false;
 
   for (unsigned i = 0; i != NumElts/2; ++i)
     if (!isUndefOrEqual(Mask[i], 0))
       return false;
   for (unsigned i = NumElts/2; i != NumElts; ++i)
     if (!isUndefOrEqual(Mask[i], NumElts/2))
       return false;
   return true;
 }

static bool isMOVHLPS_v_undef_Mask	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, <2, 3, 2, 3>

Definition at line 3944 of file X86ISelLowering.cpp.

                                                                {
   if (!VT.is128BitVector())
     return false;
 
   unsigned NumElems = VT.getVectorNumElements();
 
   if (NumElems != 4)
     return false;
 
   return isUndefOrEqual(Mask[0], 2) &&
          isUndefOrEqual(Mask[1], 3) &&
          isUndefOrEqual(Mask[2], 2) &&
          isUndefOrEqual(Mask[3], 3);
 }

static bool isMOVHLPSMask	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to MOVHLPS.

Definition at line 3925 of file X86ISelLowering.cpp.

                                                       {
   if (!VT.is128BitVector())
     return false;
 
   unsigned NumElems = VT.getVectorNumElements();
 
   if (NumElems != 4)
     return false;
 
   // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
   return isUndefOrEqual(Mask[0], 6) &&
          isUndefOrEqual(Mask[1], 7) &&
          isUndefOrEqual(Mask[2], 2) &&
          isUndefOrEqual(Mask[3], 3);
 }

static bool isMOVLHPSMask	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

isMOVLHPSMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to MOVLHPS.

Definition at line 3983 of file X86ISelLowering.cpp.

                                                       {
   if (!VT.is128BitVector())
     return false;
 
   unsigned NumElems = VT.getVectorNumElements();
 
   if (NumElems != 2 && NumElems != 4)
     return false;
 
   for (unsigned i = 0, e = NumElems/2; i != e; ++i)
     if (!isUndefOrEqual(Mask[i], i))
       return false;
 
   for (unsigned i = 0, e = NumElems/2; i != e; ++i)
     if (!isUndefOrEqual(Mask[i + e], i + NumElems))
       return false;
 
   return true;
 }

static bool isMOVLMask	(	ArrayRef< int >	Mask,
		EVT	VT
	)

static

isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to MOVSS, MOVSD, and MOVD, i.e. setting the lowest element.

Definition at line 4279 of file X86ISelLowering.cpp.

                                                    {
   if (VT.getVectorElementType().getSizeInBits() < 32)
     return false;
   if (!VT.is128BitVector())
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
 
   if (!isUndefOrEqual(Mask[0], NumElts))
     return false;
 
   for (unsigned i = 1; i != NumElts; ++i)
     if (!isUndefOrEqual(Mask[i], i))
       return false;
 
   return true;
 }

static bool isMOVLPMask	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.

Definition at line 3961 of file X86ISelLowering.cpp.

                                                     {
   if (!VT.is128BitVector())
     return false;
 
   unsigned NumElems = VT.getVectorNumElements();
 
   if (NumElems != 2 && NumElems != 4)
     return false;
 
   for (unsigned i = 0, e = NumElems/2; i != e; ++i)
     if (!isUndefOrEqual(Mask[i], i + NumElems))
       return false;
 
   for (unsigned i = NumElems/2, e = NumElems; i != e; ++i)
     if (!isUndefOrEqual(Mask[i], i))
       return false;
 
   return true;
 }

static bool isMOVSHDUPMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		const X86Subtarget *	Subtarget
	)

static

isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to MOVSHDUP. Masks to match: <1, 1, 3, 3> or <1, 1, 3, 3, 5, 5, 7, 7>

Definition at line 4457 of file X86ISelLowering.cpp.

                                                           {
   if (!Subtarget->hasSSE3())
     return false;
 
   unsigned NumElems = VT.getVectorNumElements();
 
   if ((VT.is128BitVector() && NumElems != 4) ||
       (VT.is256BitVector() && NumElems != 8) ||
       (VT.is512BitVector() && NumElems != 16))
     return false;
 
   // "i+1" is the value the indexed mask element must have
   for (unsigned i = 0; i != NumElems; i += 2)
     if (!isUndefOrEqual(Mask[i], i+1) ||
         !isUndefOrEqual(Mask[i+1], i+1))
       return false;
 
   return true;
 }

static bool isMOVSLDUPMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		const X86Subtarget *	Subtarget
	)

static

isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to MOVSLDUP. Masks to match: <0, 0, 2, 2> or <0, 0, 2, 2, 4, 4, 6, 6>

Definition at line 4481 of file X86ISelLowering.cpp.

                                                           {
   if (!Subtarget->hasSSE3())
     return false;
 
   unsigned NumElems = VT.getVectorNumElements();
 
   if ((VT.is128BitVector() && NumElems != 4) ||
       (VT.is256BitVector() && NumElems != 8) ||
       (VT.is512BitVector() && NumElems != 16))
     return false;
 
   // "i" is the value the indexed mask element must have
   for (unsigned i = 0; i != NumElems; i += 2)
     if (!isUndefOrEqual(Mask[i], i) ||
         !isUndefOrEqual(Mask[i+1], i))
       return false;
 
   return true;
 }

static bool isNoopShuffleMask ( ArrayRef< int > Mask )

static

Tiny helper function to identify a no-op mask.

This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.

Definition at line 6898 of file X86ISelLowering.cpp.

                                                   {
   for (int i = 0, Size = Mask.size(); i < Size; ++i)
     if (Mask[i] != -1 && Mask[i] != i)
       return false;
   return true;
 }

static bool isPALIGNRMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		const X86Subtarget *	Subtarget
	)

static

isPALIGNRMask - Return true if the node specifies a shuffle of elements that is suitable for input to PALIGNR.

Definition at line 3776 of file X86ISelLowering.cpp.

                                                          {
   if ((VT.is128BitVector() && !Subtarget->hasSSSE3()) ||
       (VT.is256BitVector() && !Subtarget->hasInt256()))
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
   unsigned NumLanes = VT.is512BitVector() ? 1: VT.getSizeInBits()/128;
   unsigned NumLaneElts = NumElts/NumLanes;
 
   // Do not handle 64-bit element shuffles with palignr.
   if (NumLaneElts == 2)
     return false;
 
   for (unsigned l = 0; l != NumElts; l+=NumLaneElts) {
     unsigned i;
     for (i = 0; i != NumLaneElts; ++i) {
       if (Mask[i+l] >= 0)
         break;
     }
 
     // Lane is all undef, go to next lane
     if (i == NumLaneElts)
       continue;
 
     int Start = Mask[i+l];
 
     // Make sure its in this lane in one of the sources
     if (!isUndefOrInRange(Start, l, l+NumLaneElts) &&
         !isUndefOrInRange(Start, l+NumElts, l+NumElts+NumLaneElts))
       return false;
 
     // If not lane 0, then we must match lane 0
     if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Start, Mask[i]+l))
       return false;
 
     // Correct second source to be contiguous with first source
     if (Start >= (int)NumElts)
       Start -= NumElts - NumLaneElts;
 
     // Make sure we're shifting in the right direction.
     if (Start <= (int)(i+l))
       return false;
 
     Start -= i;
 
     // Check the rest of the elements to see if they are consecutive.
     for (++i; i != NumLaneElts; ++i) {
       int Idx = Mask[i+l];
 
       // Make sure its in this lane
       if (!isUndefOrInRange(Idx, l, l+NumLaneElts) &&
           !isUndefOrInRange(Idx, l+NumElts, l+NumElts+NumLaneElts))
         return false;
 
       // If not lane 0, then we must match lane 0
       if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Idx, Mask[i]+l))
         return false;
 
       if (Idx >= (int)NumElts)
         Idx -= NumElts - NumLaneElts;
 
       if (!isUndefOrEqual(Idx, Start+i))
         return false;
 
     }
   }
 
   return true;
 }

static bool isPermImmMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		unsigned &	Imm8
	)

static

Definition at line 4357 of file X86ISelLowering.cpp.

                                                                       {
   unsigned EltSize = VT.getVectorElementType().getSizeInBits();
   if (EltSize < 32)
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
   Imm8 = 0;
   if (VT.is128BitVector() || (VT.is256BitVector() && EltSize == 64)) {
     for (unsigned i = 0; i != NumElts; ++i) {
       if (Mask[i] < 0)
         continue;
       Imm8 |= Mask[i] << (i*2);
     }
     return true;
   }
 
   unsigned LaneSize = 4;
   SmallVector<int, 4> MaskVal(LaneSize, -1);
 
   for (unsigned l = 0; l != NumElts; l += LaneSize) {
     for (unsigned i = 0; i != LaneSize; ++i) {
       if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize))
         return false;
       if (Mask[i+l] < 0)
         continue;
       if (MaskVal[i] < 0) {
         MaskVal[i] = Mask[i+l] - l;
         Imm8 |= MaskVal[i] << (i*2);
         continue;
       }
       if (Mask[i+l] != (signed)(MaskVal[i]+l))
         return false;
     }
   }
   return true;
 }

static bool isPSHUFDMask	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

isPSHUFDMask - Return true if the node specifies a shuffle of elements that is suitable for input to PSHUFD or PSHUFW. That is, it doesn't reference the second operand.

Definition at line 3708 of file X86ISelLowering.cpp.

                                                      {
   if (VT == MVT::v4f32 || VT == MVT::v4i32 )
     return (Mask[0] < 4 && Mask[1] < 4 && Mask[2] < 4 && Mask[3] < 4);
   if (VT == MVT::v2f64 || VT == MVT::v2i64)
     return (Mask[0] < 2 && Mask[1] < 2);
   return false;
 }

static bool isPSHUFHWMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	HasInt256
	)

static

isPSHUFHWMask - Return true if the node specifies a shuffle of elements that is suitable for input to PSHUFHW.

Definition at line 3718 of file X86ISelLowering.cpp.

                                                                       {
   if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16))
     return false;
 
   // Lower quadword copied in order or undef.
   if (!isSequentialOrUndefInRange(Mask, 0, 4, 0))
     return false;
 
   // Upper quadword shuffled.
   for (unsigned i = 4; i != 8; ++i)
     if (!isUndefOrInRange(Mask[i], 4, 8))
       return false;
 
   if (VT == MVT::v16i16) {
     // Lower quadword copied in order or undef.
     if (!isSequentialOrUndefInRange(Mask, 8, 4, 8))
       return false;
 
     // Upper quadword shuffled.
     for (unsigned i = 12; i != 16; ++i)
       if (!isUndefOrInRange(Mask[i], 12, 16))
         return false;
   }
 
   return true;
 }

static bool isPSHUFLWMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	HasInt256
	)

static

isPSHUFLWMask - Return true if the node specifies a shuffle of elements that is suitable for input to PSHUFLW.

Definition at line 3747 of file X86ISelLowering.cpp.

                                                                       {
   if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16))
     return false;
 
   // Upper quadword copied in order.
   if (!isSequentialOrUndefInRange(Mask, 4, 4, 4))
     return false;
 
   // Lower quadword shuffled.
   for (unsigned i = 0; i != 4; ++i)
     if (!isUndefOrInRange(Mask[i], 0, 4))
       return false;
 
   if (VT == MVT::v16i16) {
     // Upper quadword copied in order.
     if (!isSequentialOrUndefInRange(Mask, 12, 4, 12))
       return false;
 
     // Lower quadword shuffled.
     for (unsigned i = 8; i != 12; ++i)
       if (!isUndefOrInRange(Mask[i], 8, 12))
         return false;
   }
 
   return true;
 }

static bool isScalarLoadToVector	(	SDNode *	N,
		LoadSDNode **	LD = `nullptr`
	)

static

isScalarLoadToVector - Returns true if the node is a scalar load that is promoted to a vector. It also returns the LoadSDNode by reference if required.

Definition at line 4791 of file X86ISelLowering.cpp.

                                                                        {
   if (N->getOpcode() != ISD::SCALAR_TO_VECTOR)
     return false;
   N = N->getOperand(0).getNode();
   if (!ISD::isNON_EXTLoad(N))
     return false;
   if (LD)
     *LD = cast<LoadSDNode>(N);
   return true;
 }

static bool isSequentialOrUndefInRange	(	ArrayRef< int >	Mask,
		unsigned	Pos,
		unsigned	Size,
		int	Low
	)

static

isSequentialOrUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (L, L+Pos]. or is undef.

Definition at line 3697 of file X86ISelLowering.cpp.

                                                                              {
   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
     if (!isUndefOrEqual(Mask[i], Low))
       return false;
   return true;
 }

static bool isShuffleHigh128VectorInsertLow ( ShuffleVectorSDNode * SVOp )

static

isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the same as extracting the high 128-bit part of 256-bit vector and then inserting the result into the low part of a new 256-bit vector

Definition at line 18340 of file X86ISelLowering.cpp.

                                                                        {
   EVT VT = SVOp->getValueType(0);
   unsigned NumElems = VT.getVectorNumElements();
 
   // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
   for (unsigned i = 0, j = NumElems/2; i != NumElems/2; ++i, ++j)
     if (!isUndefOrEqual(SVOp->getMaskElt(i), j) ||
         SVOp->getMaskElt(j) >= 0)
       return false;
 
   return true;
 }

static bool isShuffleLow128VectorInsertHigh ( ShuffleVectorSDNode * SVOp )

static

isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the same as extracting the low 128-bit part of 256-bit vector and then inserting the result into the high part of a new 256-bit vector

Definition at line 18356 of file X86ISelLowering.cpp.

                                                                        {
   EVT VT = SVOp->getValueType(0);
   unsigned NumElems = VT.getVectorNumElements();
 
   // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
   for (unsigned i = NumElems/2, j = 0; i != NumElems; ++i, ++j)
     if (!isUndefOrEqual(SVOp->getMaskElt(i), j) ||
         SVOp->getMaskElt(j) >= 0)
       return false;
 
   return true;
 }

static bool isShuffleMaskConsecutive	(	ShuffleVectorSDNode *	SVOp,
		unsigned	MaskI,
		unsigned	MaskE,
		unsigned	OpIdx,
		unsigned	NumElems,
		unsigned &	OpNum
	)

static

isShuffleMaskConsecutive - Check if the shuffle mask indicies [MaskI, MaskE) correspond consecutively to elements from one of the vector operands, starting from its index OpIdx. Also tell OpNum which source vector operand.

Definition at line 5280 of file X86ISelLowering.cpp.

                                                                   {
   bool SeenV1 = false;
   bool SeenV2 = false;
 
   for (unsigned i = MaskI; i != MaskE; ++i, ++OpIdx) {
     int Idx = SVOp->getMaskElt(i);
     // Ignore undef indicies
     if (Idx < 0)
       continue;
 
     if (Idx < (int)NumElems)
       SeenV1 = true;
     else
       SeenV2 = true;
 
     // Only accept consecutive elements from the same vector
     if ((Idx % NumElems != OpIdx) || (SeenV1 && SeenV2))
       return false;
   }
 
   OpNum = SeenV1 ? 0 : 1;
   return true;
 }

static bool isSHUFPMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	Commuted = `false`
	)

static

isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to 128/256-bit SHUFPS and SHUFPD. If Commuted is true, then it checks for sources to be reverse of what x86 shuffles want.

Definition at line 3866 of file X86ISelLowering.cpp.

                                                                            {
 
   unsigned NumElems = VT.getVectorNumElements();
   unsigned NumLanes = VT.getSizeInBits()/128;
   unsigned NumLaneElems = NumElems/NumLanes;
 
   if (NumLaneElems != 2 && NumLaneElems != 4)
     return false;
 
   unsigned EltSize = VT.getVectorElementType().getSizeInBits();
   bool symetricMaskRequired =
     (VT.getSizeInBits() >= 256) && (EltSize == 32);
 
   // VSHUFPSY divides the resulting vector into 4 chunks.
   // The sources are also splitted into 4 chunks, and each destination
   // chunk must come from a different source chunk.
   //
   //  SRC1 =>   X7    X6    X5    X4    X3    X2    X1    X0
   //  SRC2 =>   Y7    Y6    Y5    Y4    Y3    Y2    Y1    Y9
   //
   //  DST  =>  Y7..Y4,   Y7..Y4,   X7..X4,   X7..X4,
   //           Y3..Y0,   Y3..Y0,   X3..X0,   X3..X0
   //
   // VSHUFPDY divides the resulting vector into 4 chunks.
   // The sources are also splitted into 4 chunks, and each destination
   // chunk must come from a different source chunk.
   //
   //  SRC1 =>      X3       X2       X1       X0
   //  SRC2 =>      Y3       Y2       Y1       Y0
   //
   //  DST  =>  Y3..Y2,  X3..X2,  Y1..Y0,  X1..X0
   //
   SmallVector<int, 4> MaskVal(NumLaneElems, -1);
   unsigned HalfLaneElems = NumLaneElems/2;
   for (unsigned l = 0; l != NumElems; l += NumLaneElems) {
     for (unsigned i = 0; i != NumLaneElems; ++i) {
       int Idx = Mask[i+l];
       unsigned RngStart = l + ((Commuted == (i<HalfLaneElems)) ? NumElems : 0);
       if (!isUndefOrInRange(Idx, RngStart, RngStart+NumLaneElems))
         return false;
       // For VSHUFPSY, the mask of the second half must be the same as the
       // first but with the appropriate offsets. This works in the same way as
       // VPERMILPS works with masks.
       if (!symetricMaskRequired || Idx < 0)
         continue;
       if (MaskVal[i] < 0) {
         MaskVal[i] = Idx - l;
         continue;
       }
       if ((signed)(Idx - l) != MaskVal[i])
         return false;
     }
   }
 
   return true;
 }

static bool isSingleInputShuffleMask ( ArrayRef< int > Mask )

static

Helper function to classify a mask as a single-input mask.

This isn't a generic single-input test because in the vector shuffle lowering we canonicalize single inputs to be the first input operand. This means we can more quickly test for a single input by only checking whether an input from the second operand exists. We also assume that the size of mask corresponds to the size of the input vectors which isn't true in the fully general case.

Definition at line 6913 of file X86ISelLowering.cpp.

                                                          {
   for (int M : Mask)
     if (M >= (int)Mask.size())
       return false;
   return true;
 }

static bool IsTailCallConvention ( CallingConv::ID CC )

static

IsTailCallConvention - Return true if the calling convention is one that supports tail call optimization.

Definition at line 2172 of file X86ISelLowering.cpp.

                                                    {
   return (CC == CallingConv::Fast || CC == CallingConv::GHC ||
           CC == CallingConv::HiPE);
 }

static bool isTargetShuffle ( unsigned Opcode )

static

Definition at line 3378 of file X86ISelLowering.cpp.

                                              {
   switch(Opcode) {
   default: return false;
   case X86ISD::PSHUFD:
   case X86ISD::PSHUFHW:
   case X86ISD::PSHUFLW:
   case X86ISD::SHUFP:
   case X86ISD::PALIGNR:
   case X86ISD::MOVLHPS:
   case X86ISD::MOVLHPD:
   case X86ISD::MOVHLPS:
   case X86ISD::MOVLPS:
   case X86ISD::MOVLPD:
   case X86ISD::MOVSHDUP:
   case X86ISD::MOVSLDUP:
   case X86ISD::MOVDDUP:
   case X86ISD::MOVSS:
   case X86ISD::MOVSD:
   case X86ISD::UNPCKL:
   case X86ISD::UNPCKH:
   case X86ISD::VPERMILP:
   case X86ISD::VPERM2X128:
   case X86ISD::VPERMI:
     return true;
   }
 }

static bool isTruncWithZeroHighBitsInput	(	SDValue	V,
		SelectionDAG &	DAG
	)

static

Definition at line 12506 of file X86ISelLowering.cpp.

                                                                        {
   if (V.getOpcode() != ISD::TRUNCATE)
     return false;
 
   SDValue VOp0 = V.getOperand(0);
   unsigned InBits = VOp0.getValueSizeInBits();
   unsigned Bits = V.getValueSizeInBits();
   return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
 }

static bool isUndefOrEqual	(	int	Val,
		int	CmpVal
	)

static

isUndefOrEqual - Val is either less than zero (undef) or equal to the specified value.

Definition at line 3690 of file X86ISelLowering.cpp.

                                                 {
   return (Val < 0 || Val == CmpVal);
 }

static bool isUndefOrInRange	(	int	Val,
		int	Low,
		int	Hi
	)

static

isUndefOrInRange - Return true if Val is undef or if its value falls within the specified range (L, H].

Definition at line 3684 of file X86ISelLowering.cpp.

                                                        {
   return (Val < 0) || (Val >= Low && Val < Hi);
 }

static bool isUNPCKH_v_undef_Mask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	HasInt256
	)

static

isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, <2, 2, 3, 3>

Definition at line 4222 of file X86ISelLowering.cpp.

                                                                               {
   unsigned NumElts = VT.getVectorNumElements();
 
   if (VT.is512BitVector())
     return false;
 
   assert((VT.is128BitVector() || VT.is256BitVector()) &&
          "Unsupported vector type for unpckh");
 
   if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
       (!HasInt256 || (NumElts != 16 && NumElts != 32)))
     return false;
 
   // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
   // independently on 128-bit lanes.
   unsigned NumLanes = VT.getSizeInBits()/128;
   unsigned NumLaneElts = NumElts/NumLanes;
 
   for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
     for (unsigned i = 0, j = l+NumLaneElts/2; i != NumLaneElts; i += 2, ++j) {
       int BitI  = Mask[l+i];
       int BitI1 = Mask[l+i+1];
       if (!isUndefOrEqual(BitI, j))
         return false;
       if (!isUndefOrEqual(BitI1, j))
         return false;
     }
   }
   return true;
 }

static bool isUNPCKHMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	HasInt256,
		bool	V2IsSplat = `false`
	)

static

isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to UNPCKH.

Definition at line 4131 of file X86ISelLowering.cpp.

                                                                  {
   assert(VT.getSizeInBits() >= 128 &&
          "Unsupported vector type for unpckh");
 
   // AVX defines UNPCK* to operate independently on 128-bit lanes.
   unsigned NumLanes;
   unsigned NumOf256BitLanes;
   unsigned NumElts = VT.getVectorNumElements();
   if (VT.is256BitVector()) {
     if (NumElts != 4 && NumElts != 8 &&
         (!HasInt256 || (NumElts != 16 && NumElts != 32)))
     return false;
     NumLanes = 2;
     NumOf256BitLanes = 1;
   } else if (VT.is512BitVector()) {
     assert(VT.getScalarType().getSizeInBits() >= 32 &&
            "Unsupported vector type for unpckh");
     NumLanes = 2;
     NumOf256BitLanes = 2;
   } else {
     NumLanes = 1;
     NumOf256BitLanes = 1;
   }
 
   unsigned NumEltsInStride = NumElts/NumOf256BitLanes;
   unsigned NumLaneElts = NumEltsInStride/NumLanes;
 
   for (unsigned l256 = 0; l256 < NumOf256BitLanes; l256 += 1) {
     for (unsigned l = 0; l != NumEltsInStride; l += NumLaneElts) {
       for (unsigned i = 0, j = l+NumLaneElts/2; i != NumLaneElts; i += 2, ++j) {
         int BitI  = Mask[l256*NumEltsInStride+l+i];
         int BitI1 = Mask[l256*NumEltsInStride+l+i+1];
         if (!isUndefOrEqual(BitI, j+l256*NumElts))
           return false;
         if (V2IsSplat && !isUndefOrEqual(BitI1, NumElts))
           return false;
         if (!isUndefOrEqual(BitI1, j+l256*NumElts+NumEltsInStride))
           return false;
       }
     }
   }
   return true;
 }

static bool isUNPCKL_v_undef_Mask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	HasInt256
	)

static

isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, <0, 0, 1, 1>

Definition at line 4179 of file X86ISelLowering.cpp.

                                                                               {
   unsigned NumElts = VT.getVectorNumElements();
   bool Is256BitVec = VT.is256BitVector();
 
   if (VT.is512BitVector())
     return false;
   assert((VT.is128BitVector() || VT.is256BitVector()) &&
          "Unsupported vector type for unpckh");
 
   if (Is256BitVec && NumElts != 4 && NumElts != 8 &&
       (!HasInt256 || (NumElts != 16 && NumElts != 32)))
     return false;
 
   // For 256-bit i64/f64, use MOVDDUPY instead, so reject the matching pattern
   // FIXME: Need a better way to get rid of this, there's no latency difference
   // between UNPCKLPD and MOVDDUP, the later should always be checked first and
   // the former later. We should also remove the "_undef" special mask.
   if (NumElts == 4 && Is256BitVec)
     return false;
 
   // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
   // independently on 128-bit lanes.
   unsigned NumLanes = VT.getSizeInBits()/128;
   unsigned NumLaneElts = NumElts/NumLanes;
 
   for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
     for (unsigned i = 0, j = l; i != NumLaneElts; i += 2, ++j) {
       int BitI  = Mask[l+i];
       int BitI1 = Mask[l+i+1];
 
       if (!isUndefOrEqual(BitI, j))
         return false;
       if (!isUndefOrEqual(BitI1, j))
         return false;
     }
   }
 
   return true;
 }

static bool isUNPCKLMask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	HasInt256,
		bool	V2IsSplat = `false`
	)

static

isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to UNPCKL.

Definition at line 4083 of file X86ISelLowering.cpp.

                                                                  {
 
   assert(VT.getSizeInBits() >= 128 &&
          "Unsupported vector type for unpckl");
 
   // AVX defines UNPCK* to operate independently on 128-bit lanes.
   unsigned NumLanes;
   unsigned NumOf256BitLanes;
   unsigned NumElts = VT.getVectorNumElements();
   if (VT.is256BitVector()) {
     if (NumElts != 4 && NumElts != 8 &&
         (!HasInt256 || (NumElts != 16 && NumElts != 32)))
     return false;
     NumLanes = 2;
     NumOf256BitLanes = 1;
   } else if (VT.is512BitVector()) {
     assert(VT.getScalarType().getSizeInBits() >= 32 &&
            "Unsupported vector type for unpckh");
     NumLanes = 2;
     NumOf256BitLanes = 2;
   } else {
     NumLanes = 1;
     NumOf256BitLanes = 1;
   }
 
   unsigned NumEltsInStride = NumElts/NumOf256BitLanes;
   unsigned NumLaneElts = NumEltsInStride/NumLanes;
 
   for (unsigned l256 = 0; l256 < NumOf256BitLanes; l256 += 1) {
     for (unsigned l = 0; l != NumEltsInStride; l += NumLaneElts) {
       for (unsigned i = 0, j = l; i != NumLaneElts; i += 2, ++j) {
         int BitI  = Mask[l256*NumEltsInStride+l+i];
         int BitI1 = Mask[l256*NumEltsInStride+l+i+1];
         if (!isUndefOrEqual(BitI, j+l256*NumElts))
           return false;
         if (V2IsSplat && !isUndefOrEqual(BitI1, NumElts))
           return false;
         if (!isUndefOrEqual(BitI1, j+l256*NumElts+NumEltsInStride))
           return false;
       }
     }
   }
   return true;
 }

static bool isVectorShift	(	ShuffleVectorSDNode *	SVOp,
		SelectionDAG &	DAG,
		bool &	isLeft,
		SDValue &	ShVal,
		unsigned &	ShAmt
	)

static

isVectorShift - Returns true if the shuffle can be implemented as a logical left or right shift of a vector.

Definition at line 5378 of file X86ISelLowering.cpp.

                                                                          {
   // Although the logic below support any bitwidth size, there are no
   // shift instructions which handle more than 128-bit vectors.
   if (!SVOp->getSimpleValueType(0).is128BitVector())
     return false;
 
   if (isVectorShiftLeft(SVOp, DAG, isLeft, ShVal, ShAmt) ||
       isVectorShiftRight(SVOp, DAG, isLeft, ShVal, ShAmt))
     return true;
 
   return false;
 }

static bool isVectorShiftLeft	(	ShuffleVectorSDNode *	SVOp,
		SelectionDAG &	DAG,
		bool &	isLeft,
		SDValue &	ShVal,
		unsigned &	ShAmt
	)

static

isVectorShiftLeft - Returns true if the shuffle can be implemented as a logical left shift of a vector.

Definition at line 5343 of file X86ISelLowering.cpp.

                                                                              {
   unsigned NumElems =
     SVOp->getSimpleValueType(0).getVectorNumElements();
   unsigned NumZeros = getNumOfConsecutiveZeros(
       SVOp, NumElems, true /* check zeros from left */, DAG,
       NumElems - SVOp->getMaskElt(NumElems - 1) - 1);
   unsigned OpSrc;
 
   if (!NumZeros)
     return false;
 
   // Considering the elements in the mask that are not consecutive zeros,
   // check if they consecutively come from only one of the source vectors.
   //
   //                           0    { A, B, X, X } = V2
   //                          / \    /  /
   //   vector_shuffle V1, V2 <X, X, 4, 5>
   //
   if (!isShuffleMaskConsecutive(SVOp,
             NumZeros,     // Mask Start Index
             NumElems,     // Mask End Index(exclusive)
             0,            // Where to start looking in the src vector
             NumElems,     // Number of elements in vector
             OpSrc))       // Which source operand ?
     return false;
 
   isLeft = true;
   ShAmt = NumZeros;
   ShVal = SVOp->getOperand(OpSrc);
   return true;
 }

static bool isVectorShiftRight	(	ShuffleVectorSDNode *	SVOp,
		SelectionDAG &	DAG,
		bool &	isLeft,
		SDValue &	ShVal,
		unsigned &	ShAmt
	)

static

isVectorShiftRight - Returns true if the shuffle can be implemented as a logical left shift of a vector.

Definition at line 5308 of file X86ISelLowering.cpp.

                                                                               {
   unsigned NumElems =
     SVOp->getSimpleValueType(0).getVectorNumElements();
   unsigned NumZeros = getNumOfConsecutiveZeros(
       SVOp, NumElems, false /* check zeros from right */, DAG,
       SVOp->getMaskElt(0));
   unsigned OpSrc;
 
   if (!NumZeros)
     return false;
 
   // Considering the elements in the mask that are not consecutive zeros,
   // check if they consecutively come from only one of the source vectors.
   //
   //               V1 = {X, A, B, C}     0
   //                         \  \  \    /
   //   vector_shuffle V1, V2 <1, 2, 3, X>
   //
   if (!isShuffleMaskConsecutive(SVOp,
             0,                   // Mask Start Index
             NumElems-NumZeros,   // Mask End Index(exclusive)
             NumZeros,            // Where to start looking in the src vector
             NumElems,            // Number of elements in vector
             OpSrc))              // Which source operand ?
     return false;
 
   isLeft = false;
   ShAmt = NumZeros;
   ShVal = SVOp->getOperand(OpSrc);
   return true;
 }

static bool isVEXTRACTIndex	(	SDNode *	N,
		unsigned	vecWidth
	)

static

isVEXTRACTIndex - Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors

Definition at line 4542 of file X86ISelLowering.cpp.

                                                           {
   assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width");
   if (!isa<ConstantSDNode>(N->getOperand(1).getNode()))
     return false;
 
   // The index should be aligned on a vecWidth-bit boundary.
   uint64_t Index =
     cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
 
   MVT VT = N->getSimpleValueType(0);
   unsigned ElSize = VT.getVectorElementType().getSizeInBits();
   bool Result = (Index * ElSize) % vecWidth == 0;
 
   return Result;
 }

static bool isVINSERTIndex	(	SDNode *	N,
		unsigned	vecWidth
	)

static

isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors

Definition at line 4561 of file X86ISelLowering.cpp.

                                                          {
   assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width");
   if (!isa<ConstantSDNode>(N->getOperand(2).getNode()))
     return false;
   // The index should be aligned on a vecWidth-bit boundary.
   uint64_t Index =
     cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
 
   MVT VT = N->getSimpleValueType(0);
   unsigned ElSize = VT.getVectorElementType().getSizeInBits();
   bool Result = (Index * ElSize) % vecWidth == 0;
 
   return Result;
 }

static bool isVPERM2X128Mask	(	ArrayRef< int >	Mask,
		MVT	VT,
		bool	HasFp256
	)

static

isVPERM2X128Mask - Match 256-bit shuffles where the elements are considered as permutations between 128-bit chunks or halves. As an example: this shuffle bellow: vector_shuffle <4, 5, 6, 7, 12, 13, 14, 15> The first half comes from the second half of V1 and the second half from the the second half of V2.

Definition at line 4303 of file X86ISelLowering.cpp.

                                                                         {
   if (!HasFp256 || !VT.is256BitVector())
     return false;
 
   // The shuffle result is divided into half A and half B. In total the two
   // sources have 4 halves, namely: C, D, E, F. The final values of A and
   // B must come from C, D, E or F.
   unsigned HalfSize = VT.getVectorNumElements()/2;
   bool MatchA = false, MatchB = false;
 
   // Check if A comes from one of C, D, E, F.
   for (unsigned Half = 0; Half != 4; ++Half) {
     if (isSequentialOrUndefInRange(Mask, 0, HalfSize, Half*HalfSize)) {
       MatchA = true;
       break;
     }
   }
 
   // Check if B comes from one of C, D, E, F.
   for (unsigned Half = 0; Half != 4; ++Half) {
     if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, Half*HalfSize)) {
       MatchB = true;
       break;
     }
   }
 
   return MatchA && MatchB;
 }

static bool isVPERMILPMask	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

isVPERMILPMask - Return true if the specified VECTOR_SHUFFLE operand specifies a shuffle of elements that is suitable for input to VPERMILPD*. Note that VPERMIL mask matching is different depending whether theunderlying type is 32 or 64. In the VPERMILPS the high half of the mask should point to the same elements of the low, but to the higher half of the source. In VPERMILPD the two lanes could be shuffled independently of each other with the same restriction that lanes can't be crossed. Also handles PSHUFDY.

Definition at line 4401 of file X86ISelLowering.cpp.

                                                        {
   unsigned EltSize = VT.getVectorElementType().getSizeInBits();
   if (VT.getSizeInBits() < 256 || EltSize < 32)
     return false;
   bool symetricMaskRequired = (EltSize == 32);
   unsigned NumElts = VT.getVectorNumElements();
 
   unsigned NumLanes = VT.getSizeInBits()/128;
   unsigned LaneSize = NumElts/NumLanes;
   // 2 or 4 elements in one lane
 
   SmallVector<int, 4> ExpectedMaskVal(LaneSize, -1);
   for (unsigned l = 0; l != NumElts; l += LaneSize) {
     for (unsigned i = 0; i != LaneSize; ++i) {
       if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize))
         return false;
       if (symetricMaskRequired) {
         if (ExpectedMaskVal[i] < 0 && Mask[i+l] >= 0) {
           ExpectedMaskVal[i] = Mask[i+l] - l;
           continue;
         }
         if (!isUndefOrEqual(Mask[i+l], ExpectedMaskVal[i]+l))
           return false;
       }
     }
   }
   return true;
 }

static bool isX86CCUnsigned ( unsigned X86CC )

static

Return true if the condition is an unsigned comparison operation.

Definition at line 3535 of file X86ISelLowering.cpp.

                                             {
   switch (X86CC) {
   default: llvm_unreachable("Invalid integer condition!");
   case X86::COND_E:     return true;
   case X86::COND_G:     return false;
   case X86::COND_GE:    return false;
   case X86::COND_L:     return false;
   case X86::COND_LE:    return false;
   case X86::COND_NE:    return true;
   case X86::COND_B:     return true;
   case X86::COND_A:     return true;
   case X86::COND_BE:    return true;
   case X86::COND_AE:    return true;
   }
   llvm_unreachable("covered switch fell through?!");
 }

static bool isX86LogicalCmp ( SDValue Op )

static

Definition at line 12481 of file X86ISelLowering.cpp.

                                         {
   unsigned Opc = Op.getNode()->getOpcode();
   if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
       Opc == X86ISD::SAHF)
     return true;
   if (Op.getResNo() == 1 &&
       (Opc == X86ISD::ADD ||
        Opc == X86ISD::SUB ||
        Opc == X86ISD::ADC ||
        Opc == X86ISD::SBB ||
        Opc == X86ISD::SMUL ||
        Opc == X86ISD::UMUL ||
        Opc == X86ISD::INC ||
        Opc == X86ISD::DEC ||
        Opc == X86ISD::OR ||
        Opc == X86ISD::XOR ||
        Opc == X86ISD::AND))
     return true;
 
   if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
     return true;
 
   return false;
 }

static bool isXor1OfSetCC ( SDValue Op )

static

Definition at line 12829 of file X86ISelLowering.cpp.

                                       {
   if (Op.getOpcode() != ISD::XOR)
     return false;
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
   if (N1C && N1C->getAPIntValue() == 1) {
     return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
       Op.getOperand(0).hasOneUse();
   }
   return false;
 }

static bool isZero ( SDValue V )

static

isZero - Returns true if Elt is a constant integer zero

Definition at line 4755 of file X86ISelLowering.cpp.

                               {
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
   return C && C->isNullValue();
 }

static bool isZeroShuffle ( ShuffleVectorSDNode * N )

static

isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved to an zero vector. FIXME: move to dag combiner / method on ShuffleVectorSDNode

Definition at line 4858 of file X86ISelLowering.cpp.

                                                   {
   SDValue V1 = N->getOperand(0);
   SDValue V2 = N->getOperand(1);
   unsigned NumElems = N->getValueType(0).getVectorNumElements();
   for (unsigned i = 0; i != NumElems; ++i) {
     int Idx = N->getMaskElt(i);
     if (Idx >= (int)NumElems) {
       unsigned Opc = V2.getOpcode();
       if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode()))
         continue;
       if (Opc != ISD::BUILD_VECTOR ||
           !X86::isZeroNode(V2.getOperand(Idx-NumElems)))
         return false;
     } else if (Idx >= 0) {
       unsigned Opc = V1.getOpcode();
       if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode()))
         continue;
       if (Opc != ISD::BUILD_VECTOR ||
           !X86::isZeroNode(V1.getOperand(Idx)))
         return false;
     }
   }
   return true;
 }

static SDValue lower128BitVectorShuffle	(	SDValue	Op,
		SDValue	V1,
		SDValue	V2,
		MVT	VT,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Dispatching routine to lower various 128-bit x86 vector shuffles.

This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.

Definition at line 7874 of file X86ISelLowering.cpp.

                                                            {
   switch (VT.SimpleTy) {
   case MVT::v2i64:
     return lowerV2I64VectorShuffle(Op, V1, V2, Subtarget, DAG);
   case MVT::v2f64:
     return lowerV2F64VectorShuffle(Op, V1, V2, Subtarget, DAG);
   case MVT::v4i32:
     return lowerV4I32VectorShuffle(Op, V1, V2, Subtarget, DAG);
   case MVT::v4f32:
     return lowerV4F32VectorShuffle(Op, V1, V2, Subtarget, DAG);
   case MVT::v8i16:
     return lowerV8I16VectorShuffle(Op, V1, V2, Subtarget, DAG);
   case MVT::v16i8:
     return lowerV16I8VectorShuffle(Op, V1, V2, Subtarget, DAG);
 
   default:
     llvm_unreachable("Unimplemented!");
   }
 }

static SDValue Lower256IntArith	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 14993 of file X86ISelLowering.cpp.

                                                                {
   MVT VT = Op.getSimpleValueType();
 
   assert(VT.is256BitVector() && VT.isInteger() &&
          "Unsupported value type for operation");
 
   unsigned NumElems = VT.getVectorNumElements();
   SDLoc dl(Op);
 
   // Extract the LHS vectors
   SDValue LHS = Op.getOperand(0);
   SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl);
   SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl);
 
   // Extract the RHS vectors
   SDValue RHS = Op.getOperand(1);
   SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl);
   SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl);
 
   MVT EltVT = VT.getVectorElementType();
   MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
 
   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
                      DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
                      DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
 }

static SDValue Lower256IntVSETCC	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 12072 of file X86ISelLowering.cpp.

                                                                 {
   MVT VT = Op.getSimpleValueType();
 
   assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC &&
          "Unsupported value type for operation");
 
   unsigned NumElems = VT.getVectorNumElements();
   SDLoc dl(Op);
   SDValue CC = Op.getOperand(2);
 
   // Extract the LHS vectors
   SDValue LHS = Op.getOperand(0);
   SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl);
   SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl);
 
   // Extract the RHS vectors
   SDValue RHS = Op.getOperand(1);
   SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl);
   SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl);
 
   // Issue the operation on the smaller types and concatenate the result back
   MVT EltVT = VT.getVectorElementType();
   MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
                      DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1, CC),
                      DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC));
 }

static SDValue LowerADD	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 15020 of file X86ISelLowering.cpp.

                                                        {
   assert(Op.getSimpleValueType().is256BitVector() &&
          Op.getSimpleValueType().isInteger() &&
          "Only handle AVX 256-bit vector integer operation");
   return Lower256IntArith(Op, DAG);
 }

static SDValue LowerADDC_ADDE_SUBC_SUBE	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 16128 of file X86ISelLowering.cpp.

                                                                        {
   EVT VT = Op.getNode()->getSimpleValueType(0);
 
   // Let legalize expand this if it isn't a legal type yet.
   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
     return SDValue();
 
   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
 
   unsigned Opc;
   bool ExtraOp = false;
   switch (Op.getOpcode()) {
   default: llvm_unreachable("Invalid code");
   case ISD::ADDC: Opc = X86ISD::ADD; break;
   case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break;
   case ISD::SUBC: Opc = X86ISD::SUB; break;
   case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break;
   }
 
   if (!ExtraOp)
     return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
                        Op.getOperand(1));
   return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
                      Op.getOperand(1), Op.getOperand(2));
 }

static SDValue LowerADJUST_TRAMPOLINE	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 14690 of file X86ISelLowering.cpp.

                                                                      {
   return Op.getOperand(0);
 }

static SDValue LowerANY_EXTEND	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 11154 of file X86ISelLowering.cpp.

                                                   {
   if (Subtarget->hasFp256()) {
     SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
     if (Res.getNode())
       return Res;
   }
 
   return SDValue();
 }

static SDValue LowerAsSplatVectorLoad	(	SDValue	SrcOp,
		MVT	VT,
		SDLoc	dl,
		SelectionDAG &	DAG
	)

static

Definition at line 5561 of file X86ISelLowering.cpp.

                                                                            {
 
   // Check if the scalar load can be widened into a vector load. And if
   // the address is "base + cst" see if the cst can be "absorbed" into
   // the shuffle mask.
   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) {
     SDValue Ptr = LD->getBasePtr();
     if (!ISD::isNormalLoad(LD) || LD->isVolatile())
       return SDValue();
     EVT PVT = LD->getValueType(0);
     if (PVT != MVT::i32 && PVT != MVT::f32)
       return SDValue();
 
     int FI = -1;
     int64_t Offset = 0;
     if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) {
       FI = FINode->getIndex();
       Offset = 0;
     } else if (DAG.isBaseWithConstantOffset(Ptr) &&
                isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
       FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
       Offset = Ptr.getConstantOperandVal(1);
       Ptr = Ptr.getOperand(0);
     } else {
       return SDValue();
     }
 
     // FIXME: 256-bit vector instructions don't require a strict alignment,
     // improve this code to support it better.
     unsigned RequiredAlign = VT.getSizeInBits()/8;
     SDValue Chain = LD->getChain();
     // Make sure the stack object alignment is at least 16 or 32.
     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
     if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) {
       if (MFI->isFixedObjectIndex(FI)) {
         // Can't change the alignment. FIXME: It's possible to compute
         // the exact stack offset and reference FI + adjust offset instead.
         // If someone *really* cares about this. That's the way to implement it.
         return SDValue();
       } else {
         MFI->setObjectAlignment(FI, RequiredAlign);
       }
     }
 
     // (Offset % 16 or 32) must be multiple of 4. Then address is then
     // Ptr + (Offset & ~15).
     if (Offset < 0)
       return SDValue();
     if ((Offset % RequiredAlign) & 3)
       return SDValue();
     int64_t StartOffset = Offset & ~(RequiredAlign-1);
     if (StartOffset)
       Ptr = DAG.getNode(ISD::ADD, SDLoc(Ptr), Ptr.getValueType(),
                         Ptr,DAG.getConstant(StartOffset, Ptr.getValueType()));
 
     int EltNo = (Offset - StartOffset) >> 2;
     unsigned NumElems = VT.getVectorNumElements();
 
     EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
     SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
                              LD->getPointerInfo().getWithOffset(StartOffset),
                              false, false, false, 0);
 
     SmallVector<int, 8> Mask;
     for (unsigned i = 0; i != NumElems; ++i)
       Mask.push_back(EltNo);
 
     return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), &Mask[0]);
   }
 
   return SDValue();
 }

static SDValue LowerATOMIC_FENCE	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 15959 of file X86ISelLowering.cpp.

                                                     {
   SDLoc dl(Op);
   AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
     cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
   SynchronizationScope FenceScope = static_cast<SynchronizationScope>(
     cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
 
   // The only fence that needs an instruction is a sequentially-consistent
   // cross-thread fence.
   if (FenceOrdering == SequentiallyConsistent && FenceScope == CrossThread) {
     // Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
     // no-sse2). There isn't any reason to disable it if the target processor
     // supports it.
     if (Subtarget->hasSSE2() || Subtarget->is64Bit())
       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
 
     SDValue Chain = Op.getOperand(0);
     SDValue Zero = DAG.getConstant(0, MVT::i32);
     SDValue Ops[] = {
       DAG.getRegister(X86::ESP, MVT::i32), // Base
       DAG.getTargetConstant(1, MVT::i8),   // Scale
       DAG.getRegister(0, MVT::i32),        // Index
       DAG.getTargetConstant(0, MVT::i32),  // Disp
       DAG.getRegister(0, MVT::i32),        // Segment.
       Zero,
       Chain
     };
     SDNode *Res = DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops);
     return SDValue(Res, 0);
   }
 
   // MEMBARRIER is a compiler barrier; it codegens to a no-op.
   return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
 }

static SDValue LowerATOMIC_STORE	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 16103 of file X86ISelLowering.cpp.

                                                                 {
   SDNode *Node = Op.getNode();
   SDLoc dl(Node);
   EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
 
   // Convert seq_cst store -> xchg
   // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
   // FIXME: On 32-bit, store -> fist or movq would be more efficient
   //        (The only way to get a 16-byte store is cmpxchg16b)
   // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
   if (cast<AtomicSDNode>(Node)->getOrdering() == SequentiallyConsistent ||
       !DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
     SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
                                  cast<AtomicSDNode>(Node)->getMemoryVT(),
                                  Node->getOperand(0),
                                  Node->getOperand(1), Node->getOperand(2),
                                  cast<AtomicSDNode>(Node)->getMemOperand(),
                                  cast<AtomicSDNode>(Node)->getOrdering(),
                                  cast<AtomicSDNode>(Node)->getSynchScope());
     return Swap.getValue(1);
   }
   // Other atomic stores have a simple pattern.
   return Op;
 }

static SDValue LowerAVXCONCAT_VECTORS	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 6842 of file X86ISelLowering.cpp.

                                                                      {
   SDLoc dl(Op);
   MVT ResVT = Op.getSimpleValueType();
 
   assert((ResVT.is256BitVector() ||
           ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide");
 
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   unsigned NumElems = ResVT.getVectorNumElements();
   if(ResVT.is256BitVector())
     return Concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
 
   if (Op.getNumOperands() == 4) {
     MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(),
                                 ResVT.getVectorNumElements()/2);
     SDValue V3 = Op.getOperand(2);
     SDValue V4 = Op.getOperand(3);
     return Concat256BitVectors(Concat128BitVectors(V1, V2, HalfVT, NumElems/2, DAG, dl),
       Concat128BitVectors(V3, V4, HalfVT, NumElems/2, DAG, dl), ResVT, NumElems, DAG, dl);
   }
   return Concat256BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
 }

static SDValue LowerAVXExtend	(	SDValue	Op,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 11080 of file X86ISelLowering.cpp.

                                                              {
   MVT VT = Op->getSimpleValueType(0);
   SDValue In = Op->getOperand(0);
   MVT InVT = In.getSimpleValueType();
   SDLoc dl(Op);
 
   // Optimize vectors in AVX mode:
   //
   //   v8i16 -> v8i32
   //   Use vpunpcklwd for 4 lower elements  v8i16 -> v4i32.
   //   Use vpunpckhwd for 4 upper elements  v8i16 -> v4i32.
   //   Concat upper and lower parts.
   //
   //   v4i32 -> v4i64
   //   Use vpunpckldq for 4 lower elements  v4i32 -> v2i64.
   //   Use vpunpckhdq for 4 upper elements  v4i32 -> v2i64.
   //   Concat upper and lower parts.
   //
 
   if (((VT != MVT::v16i16) || (InVT != MVT::v16i8)) &&
       ((VT != MVT::v8i32) || (InVT != MVT::v8i16)) &&
       ((VT != MVT::v4i64) || (InVT != MVT::v4i32)))
     return SDValue();
 
   if (Subtarget->hasInt256())
     return DAG.getNode(X86ISD::VZEXT, dl, VT, In);
 
   SDValue ZeroVec = getZeroVector(InVT, Subtarget, DAG, dl);
   SDValue Undef = DAG.getUNDEF(InVT);
   bool NeedZero = Op.getOpcode() == ISD::ZERO_EXTEND;
   SDValue OpLo = getUnpackl(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
   SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
 
   MVT HVT = MVT::getVectorVT(VT.getVectorElementType(),
                              VT.getVectorNumElements()/2);
 
   OpLo = DAG.getNode(ISD::BITCAST, dl, HVT, OpLo);
   OpHi = DAG.getNode(ISD::BITCAST, dl, HVT, OpHi);
 
   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
 }

static SDValue LowerBITCAST	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 16036 of file X86ISelLowering.cpp.

                                                {
   MVT SrcVT = Op.getOperand(0).getSimpleValueType();
   MVT DstVT = Op.getSimpleValueType();
 
   if (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) {
     assert(Subtarget->hasSSE2() && "Requires at least SSE2!");
     if (DstVT != MVT::f64)
       // This conversion needs to be expanded.
       return SDValue();
 
     SDValue InVec = Op->getOperand(0);
     SDLoc dl(Op);
     unsigned NumElts = SrcVT.getVectorNumElements();
     EVT SVT = SrcVT.getVectorElementType();
 
     // Widen the vector in input in the case of MVT::v2i32.
     // Example: from MVT::v2i32 to MVT::v4i32.
     SmallVector<SDValue, 16> Elts;
     for (unsigned i = 0, e = NumElts; i != e; ++i)
       Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, InVec,
                                  DAG.getIntPtrConstant(i)));
 
     // Explicitly mark the extra elements as Undef.
     SDValue Undef = DAG.getUNDEF(SVT);
     for (unsigned i = NumElts, e = NumElts * 2; i != e; ++i)
       Elts.push_back(Undef);
 
     EVT NewVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumElts * 2);
     SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Elts);
     SDValue ToV2F64 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, BV);
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, ToV2F64,
                        DAG.getIntPtrConstant(0));
   }
 
   assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() &&
          Subtarget->hasMMX() && "Unexpected custom BITCAST");
   assert((DstVT == MVT::i64 ||
           (DstVT.isVector() && DstVT.getSizeInBits()==64)) &&
          "Unexpected custom BITCAST");
   // i64 <=> MMX conversions are Legal.
   if (SrcVT==MVT::i64 && DstVT.isVector())
     return Op;
   if (DstVT==MVT::i64 && SrcVT.isVector())
     return Op;
   // MMX <=> MMX conversions are Legal.
   if (SrcVT.isVector() && DstVT.isVector())
     return Op;
   // All other conversions need to be expanded.
   return SDValue();
 }

static SDValue LowerBuildVectorv16i8	(	SDValue	Op,
		unsigned	NonZeros,
		unsigned	NumNonZero,
		unsigned	NumZero,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget,
		const TargetLowering &	TLI
	)

static

LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.

Definition at line 5394 of file X86ISelLowering.cpp.

                                                                   {
   if (NumNonZero > 8)
     return SDValue();
 
   SDLoc dl(Op);
   SDValue V;
   bool First = true;
   for (unsigned i = 0; i < 16; ++i) {
     bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
     if (ThisIsNonZero && First) {
       if (NumZero)
         V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
       else
         V = DAG.getUNDEF(MVT::v8i16);
       First = false;
     }
 
     if ((i & 1) != 0) {
       SDValue ThisElt, LastElt;
       bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0;
       if (LastIsNonZero) {
         LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl,
                               MVT::i16, Op.getOperand(i-1));
       }
       if (ThisIsNonZero) {
         ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i));
         ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16,
                               ThisElt, DAG.getConstant(8, MVT::i8));
         if (LastIsNonZero)
           ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt);
       } else
         ThisElt = LastElt;
 
       if (ThisElt.getNode())
         V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt,
                         DAG.getIntPtrConstant(i/2));
     }
   }
 
   return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V);
 }

static SDValue LowerBuildVectorv4x32	(	SDValue	Op,
		unsigned	NumElems,
		unsigned	NonZeros,
		unsigned	NumNonZero,
		unsigned	NumZero,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget,
		const TargetLowering &	TLI
	)

static

LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32.

Definition at line 5473 of file X86ISelLowering.cpp.

                                                                 {
   // We know there's at least one non-zero element
   unsigned FirstNonZeroIdx = 0;
   SDValue FirstNonZero = Op->getOperand(FirstNonZeroIdx);
   while (FirstNonZero.getOpcode() == ISD::UNDEF ||
          X86::isZeroNode(FirstNonZero)) {
     ++FirstNonZeroIdx;
     FirstNonZero = Op->getOperand(FirstNonZeroIdx);
   }
 
   if (FirstNonZero.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
       !isa<ConstantSDNode>(FirstNonZero.getOperand(1)))
     return SDValue();
 
   SDValue V = FirstNonZero.getOperand(0);
   MVT VVT = V.getSimpleValueType();
   if (!Subtarget->hasSSE41() || (VVT != MVT::v4f32 && VVT != MVT::v4i32))
     return SDValue();
 
   unsigned FirstNonZeroDst =
       cast<ConstantSDNode>(FirstNonZero.getOperand(1))->getZExtValue();
   unsigned CorrectIdx = FirstNonZeroDst == FirstNonZeroIdx;
   unsigned IncorrectIdx = CorrectIdx ? -1U : FirstNonZeroIdx;
   unsigned IncorrectDst = CorrectIdx ? -1U : FirstNonZeroDst;
 
   for (unsigned Idx = FirstNonZeroIdx + 1; Idx < NumElems; ++Idx) {
     SDValue Elem = Op.getOperand(Idx);
     if (Elem.getOpcode() == ISD::UNDEF || X86::isZeroNode(Elem))
       continue;
 
     // TODO: What else can be here? Deal with it.
     if (Elem.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
       return SDValue();
 
     // TODO: Some optimizations are still possible here
     // ex: Getting one element from a vector, and the rest from another.
     if (Elem.getOperand(0) != V)
       return SDValue();
 
     unsigned Dst = cast<ConstantSDNode>(Elem.getOperand(1))->getZExtValue();
     if (Dst == Idx)
       ++CorrectIdx;
     else if (IncorrectIdx == -1U) {
       IncorrectIdx = Idx;
       IncorrectDst = Dst;
     } else
       // There was already one element with an incorrect index.
       // We can't optimize this case to an insertps.
       return SDValue();
   }
 
   if (NumNonZero == CorrectIdx || NumNonZero == CorrectIdx + 1) {
     SDLoc dl(Op);
     EVT VT = Op.getSimpleValueType();
     unsigned ElementMoveMask = 0;
     if (IncorrectIdx == -1U)
       ElementMoveMask = FirstNonZeroIdx << 6 | FirstNonZeroIdx << 4;
     else
       ElementMoveMask = IncorrectDst << 6 | IncorrectIdx << 4;
 
     SDValue InsertpsMask =
         DAG.getIntPtrConstant(ElementMoveMask | (~NonZeros & 0xf));
     return DAG.getNode(X86ISD::INSERTPS, dl, VT, V, V, InsertpsMask);
   }
 
   return SDValue();
 }

static SDValue LowerBuildVectorv8i16	(	SDValue	Op,
		unsigned	NonZeros,
		unsigned	NumNonZero,
		unsigned	NumZero,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget,
		const TargetLowering &	TLI
	)

static

LowerBuildVectorv8i16 - Custom lower build_vector of v8i16.

Definition at line 5442 of file X86ISelLowering.cpp.

                                                                 {
   if (NumNonZero > 4)
     return SDValue();
 
   SDLoc dl(Op);
   SDValue V;
   bool First = true;
   for (unsigned i = 0; i < 8; ++i) {
     bool isNonZero = (NonZeros & (1 << i)) != 0;
     if (isNonZero) {
       if (First) {
         if (NumZero)
           V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
         else
           V = DAG.getUNDEF(MVT::v8i16);
         First = false;
       }
       V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
                       MVT::v8i16, V, Op.getOperand(i),
                       DAG.getIntPtrConstant(i));
     }
   }
 
   return V;
 }

static SDValue LowerCMP_SWAP	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 15995 of file X86ISelLowering.cpp.

                                                 {
   MVT T = Op.getSimpleValueType();
   SDLoc DL(Op);
   unsigned Reg = 0;
   unsigned size = 0;
   switch(T.SimpleTy) {
   default: llvm_unreachable("Invalid value type!");
   case MVT::i8:  Reg = X86::AL;  size = 1; break;
   case MVT::i16: Reg = X86::AX;  size = 2; break;
   case MVT::i32: Reg = X86::EAX; size = 4; break;
   case MVT::i64:
     assert(Subtarget->is64Bit() && "Node not type legal!");
     Reg = X86::RAX; size = 8;
     break;
   }
   SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
                                   Op.getOperand(2), SDValue());
   SDValue Ops[] = { cpIn.getValue(0),
                     Op.getOperand(1),
                     Op.getOperand(3),
                     DAG.getTargetConstant(size, MVT::i8),
                     cpIn.getValue(1) };
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
   MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
   SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
                                            Ops, T, MMO);
 
   SDValue cpOut =
     DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
   SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
                                       MVT::i32, cpOut.getValue(2));
   SDValue Success = DAG.getNode(X86ISD::SETCC, DL, Op->getValueType(1),
                                 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS);
 
   DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), cpOut);
   DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
   DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), EFLAGS.getValue(1));
   return SDValue();
 }

static SDValue LowerCONCAT_VECTORS	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 6866 of file X86ISelLowering.cpp.

                                                                   {
   MVT LLVM_ATTRIBUTE_UNUSED VT = Op.getSimpleValueType();
   assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
          (VT.is512BitVector() && (Op.getNumOperands() == 2 ||
           Op.getNumOperands() == 4)));
 
   // AVX can use the vinsertf128 instruction to create 256-bit vectors
   // from two other 128-bit ones.
 
   // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors
   return LowerAVXCONCAT_VECTORS(Op, DAG);
 }

static SDValue LowerCTLZ	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 14912 of file X86ISelLowering.cpp.

                                                         {
   MVT VT = Op.getSimpleValueType();
   EVT OpVT = VT;
   unsigned NumBits = VT.getSizeInBits();
   SDLoc dl(Op);
 
   Op = Op.getOperand(0);
   if (VT == MVT::i8) {
     // Zero extend to i32 since there is not an i8 bsr.
     OpVT = MVT::i32;
     Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
   }
 
   // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
   SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
   Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
 
   // If src is zero (i.e. bsr sets ZF), returns NumBits.
   SDValue Ops[] = {
     Op,
     DAG.getConstant(NumBits+NumBits-1, OpVT),
     DAG.getConstant(X86::COND_E, MVT::i8),
     Op.getValue(1)
   };
   Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
 
   // Finally xor with NumBits-1.
   Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
 
   if (VT == MVT::i8)
     Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
   return Op;
 }

static SDValue LowerCTLZ_ZERO_UNDEF	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 14946 of file X86ISelLowering.cpp.

                                                                    {
   MVT VT = Op.getSimpleValueType();
   EVT OpVT = VT;
   unsigned NumBits = VT.getSizeInBits();
   SDLoc dl(Op);
 
   Op = Op.getOperand(0);
   if (VT == MVT::i8) {
     // Zero extend to i32 since there is not an i8 bsr.
     OpVT = MVT::i32;
     Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
   }
 
   // Issue a bsr (scan bits in reverse).
   SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
   Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
 
   // And xor with NumBits-1.
   Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
 
   if (VT == MVT::i8)
     Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
   return Op;
 }

static SDValue LowerCTTZ	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 14971 of file X86ISelLowering.cpp.

                                                         {
   MVT VT = Op.getSimpleValueType();
   unsigned NumBits = VT.getSizeInBits();
   SDLoc dl(Op);
   Op = Op.getOperand(0);
 
   // Issue a bsf (scan bits forward) which also sets EFLAGS.
   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
   Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);
 
   // If src is zero (i.e. bsf sets ZF), returns NumBits.
   SDValue Ops[] = {
     Op,
     DAG.getConstant(NumBits, VT),
     DAG.getConstant(X86::COND_E, MVT::i8),
     Op.getValue(1)
   };
   return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
 }

static SDValue LowerEXTRACT_SUBVECTOR	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 10087 of file X86ISelLowering.cpp.

                                                          {
   SDLoc dl(Op);
   SDValue In =  Op.getOperand(0);
   SDValue Idx = Op.getOperand(1);
   unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
   MVT ResVT   = Op.getSimpleValueType();
   MVT InVT    = In.getSimpleValueType();
 
   if (Subtarget->hasFp256()) {
     if (ResVT.is128BitVector() &&
         (InVT.is256BitVector() || InVT.is512BitVector()) &&
         isa<ConstantSDNode>(Idx)) {
       return Extract128BitVector(In, IdxVal, DAG, dl);
     }
     if (ResVT.is256BitVector() && InVT.is512BitVector() &&
         isa<ConstantSDNode>(Idx)) {
       return Extract256BitVector(In, IdxVal, DAG, dl);
     }
   }
   return SDValue();
 }

static SDValue LowerEXTRACT_VECTOR_ELT_SSE4	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 9708 of file X86ISelLowering.cpp.

                                                                            {
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
 
   if (!Op.getOperand(0).getSimpleValueType().is128BitVector())
     return SDValue();
 
   if (VT.getSizeInBits() == 8) {
     SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
                                   Op.getOperand(0), Op.getOperand(1));
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
                                   DAG.getValueType(VT));
     return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
   }
 
   if (VT.getSizeInBits() == 16) {
     unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
     // If Idx is 0, it's cheaper to do a move instead of a pextrw.
     if (Idx == 0)
       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
                                      DAG.getNode(ISD::BITCAST, dl,
                                                  MVT::v4i32,
                                                  Op.getOperand(0)),
                                      Op.getOperand(1)));
     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
                                   Op.getOperand(0), Op.getOperand(1));
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
                                   DAG.getValueType(VT));
     return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
   }
 
   if (VT == MVT::f32) {
     // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
     // the result back to FR32 register. It's only worth matching if the
     // result has a single use which is a store or a bitcast to i32.  And in
     // the case of a store, it's not worth it if the index is a constant 0,
     // because a MOVSSmr can be used instead, which is smaller and faster.
     if (!Op.hasOneUse())
       return SDValue();
     SDNode *User = *Op.getNode()->use_begin();
     if ((User->getOpcode() != ISD::STORE ||
          (isa<ConstantSDNode>(Op.getOperand(1)) &&
           cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) &&
         (User->getOpcode() != ISD::BITCAST ||
          User->getValueType(0) != MVT::i32))
       return SDValue();
     SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
                                   DAG.getNode(ISD::BITCAST, dl, MVT::v4i32,
                                               Op.getOperand(0)),
                                               Op.getOperand(1));
     return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract);
   }
 
   if (VT == MVT::i32 || VT == MVT::i64) {
     // ExtractPS/pextrq works with constant index.
     if (isa<ConstantSDNode>(Op.getOperand(1)))
       return Op;
   }
   return SDValue();
 }

static SDValue LowerFABS	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 11378 of file X86ISelLowering.cpp.

                                                         {
   LLVMContext *Context = DAG.getContext();
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
   MVT EltVT = VT;
   unsigned NumElts = VT == MVT::f64 ? 2 : 4;
   if (VT.isVector()) {
     EltVT = VT.getVectorElementType();
     NumElts = VT.getVectorNumElements();
   }
   Constant *C;
   if (EltVT == MVT::f64)
     C = ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble,
                                           APInt(64, ~(1ULL << 63))));
   else
     C = ConstantFP::get(*Context, APFloat(APFloat::IEEEsingle,
                                           APInt(32, ~(1U << 31))));
   C = ConstantVector::getSplat(NumElts, C);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy());
   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
   SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                              MachinePointerInfo::getConstantPool(),
                              false, false, false, Alignment);
   if (VT.isVector()) {
     MVT ANDVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
     return DAG.getNode(ISD::BITCAST, dl, VT,
                        DAG.getNode(ISD::AND, dl, ANDVT,
                                    DAG.getNode(ISD::BITCAST, dl, ANDVT,
                                                Op.getOperand(0)),
                                    DAG.getNode(ISD::BITCAST, dl, ANDVT, Mask)));
   }
   return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask);
 }

static SDValue LowerFCOPYSIGN	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 11449 of file X86ISelLowering.cpp.

                                                              {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   LLVMContext *Context = DAG.getContext();
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
   MVT SrcVT = Op1.getSimpleValueType();
 
   // If second operand is smaller, extend it first.
   if (SrcVT.bitsLT(VT)) {
     Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1);
     SrcVT = VT;
   }
   // And if it is bigger, shrink it first.
   if (SrcVT.bitsGT(VT)) {
     Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1));
     SrcVT = VT;
   }
 
   // At this point the operands and the result should have the same
   // type, and that won't be f80 since that is not custom lowered.
 
   // First get the sign bit of second operand.
   SmallVector<Constant*,4> CV;
   if (SrcVT == MVT::f64) {
     const fltSemantics &Sem = APFloat::IEEEdouble;
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(64, 1ULL << 63))));
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(64, 0))));
   } else {
     const fltSemantics &Sem = APFloat::IEEEsingle;
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 1U << 31))));
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 0))));
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 0))));
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 0))));
   }
   Constant *C = ConstantVector::get(CV);
   SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16);
   SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx,
                               MachinePointerInfo::getConstantPool(),
                               false, false, false, 16);
   SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1);
 
   // Shift sign bit right or left if the two operands have different types.
   if (SrcVT.bitsGT(VT)) {
     // Op0 is MVT::f32, Op1 is MVT::f64.
     SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, SignBit);
     SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit,
                           DAG.getConstant(32, MVT::i32));
     SignBit = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, SignBit);
     SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, SignBit,
                           DAG.getIntPtrConstant(0));
   }
 
   // Clear first operand sign bit.
   CV.clear();
   if (VT == MVT::f64) {
     const fltSemantics &Sem = APFloat::IEEEdouble;
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem,
                                                    APInt(64, ~(1ULL << 63)))));
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(64, 0))));
   } else {
     const fltSemantics &Sem = APFloat::IEEEsingle;
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem,
                                                    APInt(32, ~(1U << 31)))));
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 0))));
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 0))));
     CV.push_back(ConstantFP::get(*Context, APFloat(Sem, APInt(32, 0))));
   }
   C = ConstantVector::get(CV);
   CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16);
   SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                               MachinePointerInfo::getConstantPool(),
                               false, false, false, 16);
   SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2);
 
   // Or the value with the sign bit.
   return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
 }

static SDValue LowerFGETSIGN	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 11529 of file X86ISelLowering.cpp.

                                                             {
   SDValue N0 = Op.getOperand(0);
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
 
   // Lower ISD::FGETSIGN to (AND (X86ISD::FGETSIGNx86 ...) 1).
   SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0,
                                   DAG.getConstant(1, VT));
   return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT));
 }

static SDValue LowerFNEG	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 11413 of file X86ISelLowering.cpp.

                                                         {
   LLVMContext *Context = DAG.getContext();
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
   MVT EltVT = VT;
   unsigned NumElts = VT == MVT::f64 ? 2 : 4;
   if (VT.isVector()) {
     EltVT = VT.getVectorElementType();
     NumElts = VT.getVectorNumElements();
   }
   Constant *C;
   if (EltVT == MVT::f64)
     C = ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble,
                                           APInt(64, 1ULL << 63)));
   else
     C = ConstantFP::get(*Context, APFloat(APFloat::IEEEsingle,
                                           APInt(32, 1U << 31)));
   C = ConstantVector::getSplat(NumElts, C);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy());
   unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
   SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                              MachinePointerInfo::getConstantPool(),
                              false, false, false, Alignment);
   if (VT.isVector()) {
     MVT XORVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits()/64);
     return DAG.getNode(ISD::BITCAST, dl, VT,
                        DAG.getNode(ISD::XOR, dl, XORVT,
                                    DAG.getNode(ISD::BITCAST, dl, XORVT,
                                                Op.getOperand(0)),
                                    DAG.getNode(ISD::BITCAST, dl, XORVT, Mask)));
   }
 
   return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask);
 }

static SDValue LowerFP_EXTEND	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 11365 of file X86ISelLowering.cpp.

                                                              {
   SDLoc DL(Op);
   MVT VT = Op.getSimpleValueType();
   SDValue In = Op.getOperand(0);
   MVT SVT = In.getSimpleValueType();
 
   assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!");
 
   return DAG.getNode(X86ISD::VFPEXT, DL, VT,
                      DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32,
                                  In, DAG.getUNDEF(SVT)));
 }

static SDValue LowerFSINCOS	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 16154 of file X86ISelLowering.cpp.

                                                {
   assert(Subtarget->isTargetDarwin() && Subtarget->is64Bit());
 
   // For MacOSX, we want to call an alternative entry point: __sincos_stret,
   // which returns the values as { float, float } (in XMM0) or
   // { double, double } (which is returned in XMM0, XMM1).
   SDLoc dl(Op);
   SDValue Arg = Op.getOperand(0);
   EVT ArgVT = Arg.getValueType();
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
 
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
 
   Entry.Node = Arg;
   Entry.Ty = ArgTy;
   Entry.isSExt = false;
   Entry.isZExt = false;
   Args.push_back(Entry);
 
   bool isF64 = ArgVT == MVT::f64;
   // Only optimize x86_64 for now. i386 is a bit messy. For f32,
   // the small struct {f32, f32} is returned in (eax, edx). For f64,
   // the results are returned via SRet in memory.
   const char *LibcallName =  isF64 ? "__sincos_stret" : "__sincosf_stret";
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue Callee = DAG.getExternalSymbol(LibcallName, TLI.getPointerTy());
 
   Type *RetTy = isF64
     ? (Type*)StructType::get(ArgTy, ArgTy, NULL)
     : (Type*)VectorType::get(ArgTy, 4);
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(dl).setChain(DAG.getEntryNode())
     .setCallee(CallingConv::C, RetTy, Callee, std::move(Args), 0);
 
   std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
 
   if (isF64)
     // Returned in xmm0 and xmm1.
     return CallResult.first;
 
   // Returned in bits 0:31 and 32:64 xmm0.
   SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
                                CallResult.first, DAG.getIntPtrConstant(0));
   SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
                                CallResult.first, DAG.getIntPtrConstant(1));
   SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
   return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
 }

static SDValue LowerINSERT_SUBVECTOR	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 10113 of file X86ISelLowering.cpp.

                                                         {
   if (Subtarget->hasFp256()) {
     SDLoc dl(Op.getNode());
     SDValue Vec = Op.getNode()->getOperand(0);
     SDValue SubVec = Op.getNode()->getOperand(1);
     SDValue Idx = Op.getNode()->getOperand(2);
 
     if ((Op.getNode()->getSimpleValueType(0).is256BitVector() ||
          Op.getNode()->getSimpleValueType(0).is512BitVector()) &&
         SubVec.getNode()->getSimpleValueType(0).is128BitVector() &&
         isa<ConstantSDNode>(Idx)) {
       unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
       return Insert128BitVector(Vec, SubVec, IdxVal, DAG, dl);
     }
 
     if (Op.getNode()->getSimpleValueType(0).is512BitVector() &&
         SubVec.getNode()->getSimpleValueType(0).is256BitVector() &&
         isa<ConstantSDNode>(Idx)) {
       unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
       return Insert256BitVector(Vec, SubVec, IdxVal, DAG, dl);
     }
   }
   return SDValue();
 }

static SDValue LowerINSERT_VECTOR_ELT_SSE4	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 9917 of file X86ISelLowering.cpp.

                                                                           {
   MVT VT = Op.getSimpleValueType();
   MVT EltVT = VT.getVectorElementType();
   SDLoc dl(Op);
 
   SDValue N0 = Op.getOperand(0);
   SDValue N1 = Op.getOperand(1);
   SDValue N2 = Op.getOperand(2);
 
   if (!VT.is128BitVector())
     return SDValue();
 
   if ((EltVT.getSizeInBits() == 8 || EltVT.getSizeInBits() == 16) &&
       isa<ConstantSDNode>(N2)) {
     unsigned Opc;
     if (VT == MVT::v8i16)
       Opc = X86ISD::PINSRW;
     else if (VT == MVT::v16i8)
       Opc = X86ISD::PINSRB;
     else
       Opc = X86ISD::PINSRB;
 
     // Transform it so it match pinsr{b,w} which expects a GR32 as its second
     // argument.
     if (N1.getValueType() != MVT::i32)
       N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
     if (N2.getValueType() != MVT::i32)
       N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue());
     return DAG.getNode(Opc, dl, VT, N0, N1, N2);
   }
 
   if (EltVT == MVT::f32 && isa<ConstantSDNode>(N2)) {
     // Bits [7:6] of the constant are the source select.  This will always be
     //  zero here.  The DAG Combiner may combine an extract_elt index into these
     //  bits.  For example (insert (extract, 3), 2) could be matched by putting
     //  the '3' into bits [7:6] of X86ISD::INSERTPS.
     // Bits [5:4] of the constant are the destination select.  This is the
     //  value of the incoming immediate.
     // Bits [3:0] of the constant are the zero mask.  The DAG Combiner may
     //   combine either bitwise AND or insert of float 0.0 to set these bits.
     N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue() << 4);
     // Create this as a scalar to vector..
     N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
     return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
   }
 
   if ((EltVT == MVT::i32 || EltVT == MVT::i64) && isa<ConstantSDNode>(N2)) {
     // PINSR* works with constant index.
     return Op;
   }
   return SDValue();
 }

static SDValue LowerINTRINSIC_W_CHAIN	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 14485 of file X86ISelLowering.cpp.

                                                          {
   InitIntinsicsMap();
   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
   std::map < unsigned, IntrinsicData>::const_iterator itr = IntrMap.find(IntNo);
   if (itr == IntrMap.end())
     return SDValue();
 
   SDLoc dl(Op);
   IntrinsicData Intr = itr->second;
   switch(Intr.Type) {
   case RDSEED:
   case RDRAND: {
     // Emit the node with the right value type.
     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Glue, MVT::Other);
     SDValue Result = DAG.getNode(Intr.Opc0, dl, VTs, Op.getOperand(0));
 
     // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
     // Otherwise return the value from Rand, which is always 0, casted to i32.
     SDValue Ops[] = { DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
                       DAG.getConstant(1, Op->getValueType(1)),
                       DAG.getConstant(X86::COND_B, MVT::i32),
                       SDValue(Result.getNode(), 1) };
     SDValue isValid = DAG.getNode(X86ISD::CMOV, dl,
                                   DAG.getVTList(Op->getValueType(1), MVT::Glue),
                                   Ops);
 
     // Return { result, isValid, chain }.
     return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
                        SDValue(Result.getNode(), 2));
   }
   case GATHER: {
   //gather(v1, mask, index, base, scale);
     SDValue Chain = Op.getOperand(0);
     SDValue Src   = Op.getOperand(2);
     SDValue Base  = Op.getOperand(3);
     SDValue Index = Op.getOperand(4);
     SDValue Mask  = Op.getOperand(5);
     SDValue Scale = Op.getOperand(6);
     return getGatherNode(Intr.Opc0, Op, DAG, Src, Mask, Base, Index, Scale, Chain,
                           Subtarget);
   }
   case SCATTER: {
   //scatter(base, mask, index, v1, scale);
     SDValue Chain = Op.getOperand(0);
     SDValue Base  = Op.getOperand(2);
     SDValue Mask  = Op.getOperand(3);
     SDValue Index = Op.getOperand(4);
     SDValue Src   = Op.getOperand(5);
     SDValue Scale = Op.getOperand(6);
     return getScatterNode(Intr.Opc0, Op, DAG, Src, Mask, Base, Index, Scale, Chain);
   }
   case PREFETCH: {
     SDValue Hint = Op.getOperand(6);
     unsigned HintVal;
     if (dyn_cast<ConstantSDNode> (Hint) == nullptr ||
         (HintVal = dyn_cast<ConstantSDNode> (Hint)->getZExtValue()) > 1)
       llvm_unreachable("Wrong prefetch hint in intrinsic: should be 0 or 1");
     unsigned Opcode = (HintVal ? Intr.Opc1 : Intr.Opc0);
     SDValue Chain = Op.getOperand(0);
     SDValue Mask  = Op.getOperand(2);
     SDValue Index = Op.getOperand(3);
     SDValue Base  = Op.getOperand(4);
     SDValue Scale = Op.getOperand(5);
     return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain);
   }
   // Read Time Stamp Counter (RDTSC) and Processor ID (RDTSCP).
   case RDTSC: {
     SmallVector<SDValue, 2> Results;
     getReadTimeStampCounter(Op.getNode(), dl, Intr.Opc0, DAG, Subtarget, Results);
     return DAG.getMergeValues(Results, dl);
   }
   // Read Performance Monitoring Counters.
   case RDPMC: {
     SmallVector<SDValue, 2> Results;
     getReadPerformanceCounter(Op.getNode(), dl, DAG, Subtarget, Results);
     return DAG.getMergeValues(Results, dl);
   }
   // XTEST intrinsics.
   case XTEST: {
     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
     SDValue InTrans = DAG.getNode(X86ISD::XTEST, dl, VTs, Op.getOperand(0));
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
                                 DAG.getConstant(X86::COND_NE, MVT::i8),
                                 InTrans);
     SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
     return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
                        Ret, SDValue(InTrans.getNode(), 1));
   }
   }
   llvm_unreachable("Unknown Intrinsic Type");
 }

static SDValue LowerINTRINSIC_WO_CHAIN	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 13486 of file X86ISelLowering.cpp.

                                                                       {
   SDLoc dl(Op);
   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   switch (IntNo) {
   default: return SDValue();    // Don't custom lower most intrinsics.
   // Comparison intrinsics.
   case Intrinsic::x86_sse_comieq_ss:
   case Intrinsic::x86_sse_comilt_ss:
   case Intrinsic::x86_sse_comile_ss:
   case Intrinsic::x86_sse_comigt_ss:
   case Intrinsic::x86_sse_comige_ss:
   case Intrinsic::x86_sse_comineq_ss:
   case Intrinsic::x86_sse_ucomieq_ss:
   case Intrinsic::x86_sse_ucomilt_ss:
   case Intrinsic::x86_sse_ucomile_ss:
   case Intrinsic::x86_sse_ucomigt_ss:
   case Intrinsic::x86_sse_ucomige_ss:
   case Intrinsic::x86_sse_ucomineq_ss:
   case Intrinsic::x86_sse2_comieq_sd:
   case Intrinsic::x86_sse2_comilt_sd:
   case Intrinsic::x86_sse2_comile_sd:
   case Intrinsic::x86_sse2_comigt_sd:
   case Intrinsic::x86_sse2_comige_sd:
   case Intrinsic::x86_sse2_comineq_sd:
   case Intrinsic::x86_sse2_ucomieq_sd:
   case Intrinsic::x86_sse2_ucomilt_sd:
   case Intrinsic::x86_sse2_ucomile_sd:
   case Intrinsic::x86_sse2_ucomigt_sd:
   case Intrinsic::x86_sse2_ucomige_sd:
   case Intrinsic::x86_sse2_ucomineq_sd: {
     unsigned Opc;
     ISD::CondCode CC;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_sse_comieq_ss:
     case Intrinsic::x86_sse2_comieq_sd:
       Opc = X86ISD::COMI;
       CC = ISD::SETEQ;
       break;
     case Intrinsic::x86_sse_comilt_ss:
     case Intrinsic::x86_sse2_comilt_sd:
       Opc = X86ISD::COMI;
       CC = ISD::SETLT;
       break;
     case Intrinsic::x86_sse_comile_ss:
     case Intrinsic::x86_sse2_comile_sd:
       Opc = X86ISD::COMI;
       CC = ISD::SETLE;
       break;
     case Intrinsic::x86_sse_comigt_ss:
     case Intrinsic::x86_sse2_comigt_sd:
       Opc = X86ISD::COMI;
       CC = ISD::SETGT;
       break;
     case Intrinsic::x86_sse_comige_ss:
     case Intrinsic::x86_sse2_comige_sd:
       Opc = X86ISD::COMI;
       CC = ISD::SETGE;
       break;
     case Intrinsic::x86_sse_comineq_ss:
     case Intrinsic::x86_sse2_comineq_sd:
       Opc = X86ISD::COMI;
       CC = ISD::SETNE;
       break;
     case Intrinsic::x86_sse_ucomieq_ss:
     case Intrinsic::x86_sse2_ucomieq_sd:
       Opc = X86ISD::UCOMI;
       CC = ISD::SETEQ;
       break;
     case Intrinsic::x86_sse_ucomilt_ss:
     case Intrinsic::x86_sse2_ucomilt_sd:
       Opc = X86ISD::UCOMI;
       CC = ISD::SETLT;
       break;
     case Intrinsic::x86_sse_ucomile_ss:
     case Intrinsic::x86_sse2_ucomile_sd:
       Opc = X86ISD::UCOMI;
       CC = ISD::SETLE;
       break;
     case Intrinsic::x86_sse_ucomigt_ss:
     case Intrinsic::x86_sse2_ucomigt_sd:
       Opc = X86ISD::UCOMI;
       CC = ISD::SETGT;
       break;
     case Intrinsic::x86_sse_ucomige_ss:
     case Intrinsic::x86_sse2_ucomige_sd:
       Opc = X86ISD::UCOMI;
       CC = ISD::SETGE;
       break;
     case Intrinsic::x86_sse_ucomineq_ss:
     case Intrinsic::x86_sse2_ucomineq_sd:
       Opc = X86ISD::UCOMI;
       CC = ISD::SETNE;
       break;
     }
 
     SDValue LHS = Op.getOperand(1);
     SDValue RHS = Op.getOperand(2);
     unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG);
     assert(X86CC != X86::COND_INVALID && "Unexpected illegal condition!");
     SDValue Cond = DAG.getNode(Opc, dl, MVT::i32, LHS, RHS);
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
                                 DAG.getConstant(X86CC, MVT::i8), Cond);
     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
   }
 
   // Arithmetic intrinsics.
   case Intrinsic::x86_sse2_pmulu_dq:
   case Intrinsic::x86_avx2_pmulu_dq:
     return DAG.getNode(X86ISD::PMULUDQ, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_sse41_pmuldq:
   case Intrinsic::x86_avx2_pmul_dq:
     return DAG.getNode(X86ISD::PMULDQ, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_sse2_pmulhu_w:
   case Intrinsic::x86_avx2_pmulhu_w:
     return DAG.getNode(ISD::MULHU, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_sse2_pmulh_w:
   case Intrinsic::x86_avx2_pmulh_w:
     return DAG.getNode(ISD::MULHS, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   // SSE2/AVX2 sub with unsigned saturation intrinsics
   case Intrinsic::x86_sse2_psubus_b:
   case Intrinsic::x86_sse2_psubus_w:
   case Intrinsic::x86_avx2_psubus_b:
   case Intrinsic::x86_avx2_psubus_w:
     return DAG.getNode(X86ISD::SUBUS, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   // SSE3/AVX horizontal add/sub intrinsics
   case Intrinsic::x86_sse3_hadd_ps:
   case Intrinsic::x86_sse3_hadd_pd:
   case Intrinsic::x86_avx_hadd_ps_256:
   case Intrinsic::x86_avx_hadd_pd_256:
   case Intrinsic::x86_sse3_hsub_ps:
   case Intrinsic::x86_sse3_hsub_pd:
   case Intrinsic::x86_avx_hsub_ps_256:
   case Intrinsic::x86_avx_hsub_pd_256:
   case Intrinsic::x86_ssse3_phadd_w_128:
   case Intrinsic::x86_ssse3_phadd_d_128:
   case Intrinsic::x86_avx2_phadd_w:
   case Intrinsic::x86_avx2_phadd_d:
   case Intrinsic::x86_ssse3_phsub_w_128:
   case Intrinsic::x86_ssse3_phsub_d_128:
   case Intrinsic::x86_avx2_phsub_w:
   case Intrinsic::x86_avx2_phsub_d: {
     unsigned Opcode;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_sse3_hadd_ps:
     case Intrinsic::x86_sse3_hadd_pd:
     case Intrinsic::x86_avx_hadd_ps_256:
     case Intrinsic::x86_avx_hadd_pd_256:
       Opcode = X86ISD::FHADD;
       break;
     case Intrinsic::x86_sse3_hsub_ps:
     case Intrinsic::x86_sse3_hsub_pd:
     case Intrinsic::x86_avx_hsub_ps_256:
     case Intrinsic::x86_avx_hsub_pd_256:
       Opcode = X86ISD::FHSUB;
       break;
     case Intrinsic::x86_ssse3_phadd_w_128:
     case Intrinsic::x86_ssse3_phadd_d_128:
     case Intrinsic::x86_avx2_phadd_w:
     case Intrinsic::x86_avx2_phadd_d:
       Opcode = X86ISD::HADD;
       break;
     case Intrinsic::x86_ssse3_phsub_w_128:
     case Intrinsic::x86_ssse3_phsub_d_128:
     case Intrinsic::x86_avx2_phsub_w:
     case Intrinsic::x86_avx2_phsub_d:
       Opcode = X86ISD::HSUB;
       break;
     }
     return DAG.getNode(Opcode, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   }
 
   // SSE2/SSE41/AVX2 integer max/min intrinsics.
   case Intrinsic::x86_sse2_pmaxu_b:
   case Intrinsic::x86_sse41_pmaxuw:
   case Intrinsic::x86_sse41_pmaxud:
   case Intrinsic::x86_avx2_pmaxu_b:
   case Intrinsic::x86_avx2_pmaxu_w:
   case Intrinsic::x86_avx2_pmaxu_d:
   case Intrinsic::x86_sse2_pminu_b:
   case Intrinsic::x86_sse41_pminuw:
   case Intrinsic::x86_sse41_pminud:
   case Intrinsic::x86_avx2_pminu_b:
   case Intrinsic::x86_avx2_pminu_w:
   case Intrinsic::x86_avx2_pminu_d:
   case Intrinsic::x86_sse41_pmaxsb:
   case Intrinsic::x86_sse2_pmaxs_w:
   case Intrinsic::x86_sse41_pmaxsd:
   case Intrinsic::x86_avx2_pmaxs_b:
   case Intrinsic::x86_avx2_pmaxs_w:
   case Intrinsic::x86_avx2_pmaxs_d:
   case Intrinsic::x86_sse41_pminsb:
   case Intrinsic::x86_sse2_pmins_w:
   case Intrinsic::x86_sse41_pminsd:
   case Intrinsic::x86_avx2_pmins_b:
   case Intrinsic::x86_avx2_pmins_w:
   case Intrinsic::x86_avx2_pmins_d: {
     unsigned Opcode;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_sse2_pmaxu_b:
     case Intrinsic::x86_sse41_pmaxuw:
     case Intrinsic::x86_sse41_pmaxud:
     case Intrinsic::x86_avx2_pmaxu_b:
     case Intrinsic::x86_avx2_pmaxu_w:
     case Intrinsic::x86_avx2_pmaxu_d:
       Opcode = X86ISD::UMAX;
       break;
     case Intrinsic::x86_sse2_pminu_b:
     case Intrinsic::x86_sse41_pminuw:
     case Intrinsic::x86_sse41_pminud:
     case Intrinsic::x86_avx2_pminu_b:
     case Intrinsic::x86_avx2_pminu_w:
     case Intrinsic::x86_avx2_pminu_d:
       Opcode = X86ISD::UMIN;
       break;
     case Intrinsic::x86_sse41_pmaxsb:
     case Intrinsic::x86_sse2_pmaxs_w:
     case Intrinsic::x86_sse41_pmaxsd:
     case Intrinsic::x86_avx2_pmaxs_b:
     case Intrinsic::x86_avx2_pmaxs_w:
     case Intrinsic::x86_avx2_pmaxs_d:
       Opcode = X86ISD::SMAX;
       break;
     case Intrinsic::x86_sse41_pminsb:
     case Intrinsic::x86_sse2_pmins_w:
     case Intrinsic::x86_sse41_pminsd:
     case Intrinsic::x86_avx2_pmins_b:
     case Intrinsic::x86_avx2_pmins_w:
     case Intrinsic::x86_avx2_pmins_d:
       Opcode = X86ISD::SMIN;
       break;
     }
     return DAG.getNode(Opcode, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   }
 
   // SSE/SSE2/AVX floating point max/min intrinsics.
   case Intrinsic::x86_sse_max_ps:
   case Intrinsic::x86_sse2_max_pd:
   case Intrinsic::x86_avx_max_ps_256:
   case Intrinsic::x86_avx_max_pd_256:
   case Intrinsic::x86_sse_min_ps:
   case Intrinsic::x86_sse2_min_pd:
   case Intrinsic::x86_avx_min_ps_256:
   case Intrinsic::x86_avx_min_pd_256: {
     unsigned Opcode;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_sse_max_ps:
     case Intrinsic::x86_sse2_max_pd:
     case Intrinsic::x86_avx_max_ps_256:
     case Intrinsic::x86_avx_max_pd_256:
       Opcode = X86ISD::FMAX;
       break;
     case Intrinsic::x86_sse_min_ps:
     case Intrinsic::x86_sse2_min_pd:
     case Intrinsic::x86_avx_min_ps_256:
     case Intrinsic::x86_avx_min_pd_256:
       Opcode = X86ISD::FMIN;
       break;
     }
     return DAG.getNode(Opcode, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   }
 
   // AVX2 variable shift intrinsics
   case Intrinsic::x86_avx2_psllv_d:
   case Intrinsic::x86_avx2_psllv_q:
   case Intrinsic::x86_avx2_psllv_d_256:
   case Intrinsic::x86_avx2_psllv_q_256:
   case Intrinsic::x86_avx2_psrlv_d:
   case Intrinsic::x86_avx2_psrlv_q:
   case Intrinsic::x86_avx2_psrlv_d_256:
   case Intrinsic::x86_avx2_psrlv_q_256:
   case Intrinsic::x86_avx2_psrav_d:
   case Intrinsic::x86_avx2_psrav_d_256: {
     unsigned Opcode;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_avx2_psllv_d:
     case Intrinsic::x86_avx2_psllv_q:
     case Intrinsic::x86_avx2_psllv_d_256:
     case Intrinsic::x86_avx2_psllv_q_256:
       Opcode = ISD::SHL;
       break;
     case Intrinsic::x86_avx2_psrlv_d:
     case Intrinsic::x86_avx2_psrlv_q:
     case Intrinsic::x86_avx2_psrlv_d_256:
     case Intrinsic::x86_avx2_psrlv_q_256:
       Opcode = ISD::SRL;
       break;
     case Intrinsic::x86_avx2_psrav_d:
     case Intrinsic::x86_avx2_psrav_d_256:
       Opcode = ISD::SRA;
       break;
     }
     return DAG.getNode(Opcode, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   }
 
   case Intrinsic::x86_sse2_packssdw_128:
   case Intrinsic::x86_sse2_packsswb_128:
   case Intrinsic::x86_avx2_packssdw:
   case Intrinsic::x86_avx2_packsswb:
     return DAG.getNode(X86ISD::PACKSS, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_sse2_packuswb_128:
   case Intrinsic::x86_sse41_packusdw:
   case Intrinsic::x86_avx2_packuswb:
   case Intrinsic::x86_avx2_packusdw:
     return DAG.getNode(X86ISD::PACKUS, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_ssse3_pshuf_b_128:
   case Intrinsic::x86_avx2_pshuf_b:
     return DAG.getNode(X86ISD::PSHUFB, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_sse2_pshuf_d:
     return DAG.getNode(X86ISD::PSHUFD, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_sse2_pshufl_w:
     return DAG.getNode(X86ISD::PSHUFLW, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_sse2_pshufh_w:
     return DAG.getNode(X86ISD::PSHUFHW, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_ssse3_psign_b_128:
   case Intrinsic::x86_ssse3_psign_w_128:
   case Intrinsic::x86_ssse3_psign_d_128:
   case Intrinsic::x86_avx2_psign_b:
   case Intrinsic::x86_avx2_psign_w:
   case Intrinsic::x86_avx2_psign_d:
     return DAG.getNode(X86ISD::PSIGN, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
 
   case Intrinsic::x86_sse41_insertps:
     return DAG.getNode(X86ISD::INSERTPS, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
 
   case Intrinsic::x86_avx_vperm2f128_ps_256:
   case Intrinsic::x86_avx_vperm2f128_pd_256:
   case Intrinsic::x86_avx_vperm2f128_si_256:
   case Intrinsic::x86_avx2_vperm2i128:
     return DAG.getNode(X86ISD::VPERM2X128, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
 
   case Intrinsic::x86_avx2_permd:
   case Intrinsic::x86_avx2_permps:
     // Operands intentionally swapped. Mask is last operand to intrinsic,
     // but second operand for node/instruction.
     return DAG.getNode(X86ISD::VPERMV, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(1));
 
   case Intrinsic::x86_sse_sqrt_ps:
   case Intrinsic::x86_sse2_sqrt_pd:
   case Intrinsic::x86_avx_sqrt_ps_256:
   case Intrinsic::x86_avx_sqrt_pd_256:
     return DAG.getNode(ISD::FSQRT, dl, Op.getValueType(), Op.getOperand(1));
 
   // ptest and testp intrinsics. The intrinsic these come from are designed to
   // return an integer value, not just an instruction so lower it to the ptest
   // or testp pattern and a setcc for the result.
   case Intrinsic::x86_sse41_ptestz:
   case Intrinsic::x86_sse41_ptestc:
   case Intrinsic::x86_sse41_ptestnzc:
   case Intrinsic::x86_avx_ptestz_256:
   case Intrinsic::x86_avx_ptestc_256:
   case Intrinsic::x86_avx_ptestnzc_256:
   case Intrinsic::x86_avx_vtestz_ps:
   case Intrinsic::x86_avx_vtestc_ps:
   case Intrinsic::x86_avx_vtestnzc_ps:
   case Intrinsic::x86_avx_vtestz_pd:
   case Intrinsic::x86_avx_vtestc_pd:
   case Intrinsic::x86_avx_vtestnzc_pd:
   case Intrinsic::x86_avx_vtestz_ps_256:
   case Intrinsic::x86_avx_vtestc_ps_256:
   case Intrinsic::x86_avx_vtestnzc_ps_256:
   case Intrinsic::x86_avx_vtestz_pd_256:
   case Intrinsic::x86_avx_vtestc_pd_256:
   case Intrinsic::x86_avx_vtestnzc_pd_256: {
     bool IsTestPacked = false;
     unsigned X86CC;
     switch (IntNo) {
     default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
     case Intrinsic::x86_avx_vtestz_ps:
     case Intrinsic::x86_avx_vtestz_pd:
     case Intrinsic::x86_avx_vtestz_ps_256:
     case Intrinsic::x86_avx_vtestz_pd_256:
       IsTestPacked = true; // Fallthrough
     case Intrinsic::x86_sse41_ptestz:
     case Intrinsic::x86_avx_ptestz_256:
       // ZF = 1
       X86CC = X86::COND_E;
       break;
     case Intrinsic::x86_avx_vtestc_ps:
     case Intrinsic::x86_avx_vtestc_pd:
     case Intrinsic::x86_avx_vtestc_ps_256:
     case Intrinsic::x86_avx_vtestc_pd_256:
       IsTestPacked = true; // Fallthrough
     case Intrinsic::x86_sse41_ptestc:
     case Intrinsic::x86_avx_ptestc_256:
       // CF = 1
       X86CC = X86::COND_B;
       break;
     case Intrinsic::x86_avx_vtestnzc_ps:
     case Intrinsic::x86_avx_vtestnzc_pd:
     case Intrinsic::x86_avx_vtestnzc_ps_256:
     case Intrinsic::x86_avx_vtestnzc_pd_256:
       IsTestPacked = true; // Fallthrough
     case Intrinsic::x86_sse41_ptestnzc:
     case Intrinsic::x86_avx_ptestnzc_256:
       // ZF and CF = 0
       X86CC = X86::COND_A;
       break;
     }
 
     SDValue LHS = Op.getOperand(1);
     SDValue RHS = Op.getOperand(2);
     unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST;
     SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
     SDValue CC = DAG.getConstant(X86CC, MVT::i8);
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
   }
   case Intrinsic::x86_avx512_kortestz_w:
   case Intrinsic::x86_avx512_kortestc_w: {
     unsigned X86CC = (IntNo == Intrinsic::x86_avx512_kortestz_w)? X86::COND_E: X86::COND_B;
     SDValue LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(1));
     SDValue RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(2));
     SDValue CC = DAG.getConstant(X86CC, MVT::i8);
     SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test);
     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
   }
 
   // SSE/AVX shift intrinsics
   case Intrinsic::x86_sse2_psll_w:
   case Intrinsic::x86_sse2_psll_d:
   case Intrinsic::x86_sse2_psll_q:
   case Intrinsic::x86_avx2_psll_w:
   case Intrinsic::x86_avx2_psll_d:
   case Intrinsic::x86_avx2_psll_q:
   case Intrinsic::x86_sse2_psrl_w:
   case Intrinsic::x86_sse2_psrl_d:
   case Intrinsic::x86_sse2_psrl_q:
   case Intrinsic::x86_avx2_psrl_w:
   case Intrinsic::x86_avx2_psrl_d:
   case Intrinsic::x86_avx2_psrl_q:
   case Intrinsic::x86_sse2_psra_w:
   case Intrinsic::x86_sse2_psra_d:
   case Intrinsic::x86_avx2_psra_w:
   case Intrinsic::x86_avx2_psra_d: {
     unsigned Opcode;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_sse2_psll_w:
     case Intrinsic::x86_sse2_psll_d:
     case Intrinsic::x86_sse2_psll_q:
     case Intrinsic::x86_avx2_psll_w:
     case Intrinsic::x86_avx2_psll_d:
     case Intrinsic::x86_avx2_psll_q:
       Opcode = X86ISD::VSHL;
       break;
     case Intrinsic::x86_sse2_psrl_w:
     case Intrinsic::x86_sse2_psrl_d:
     case Intrinsic::x86_sse2_psrl_q:
     case Intrinsic::x86_avx2_psrl_w:
     case Intrinsic::x86_avx2_psrl_d:
     case Intrinsic::x86_avx2_psrl_q:
       Opcode = X86ISD::VSRL;
       break;
     case Intrinsic::x86_sse2_psra_w:
     case Intrinsic::x86_sse2_psra_d:
     case Intrinsic::x86_avx2_psra_w:
     case Intrinsic::x86_avx2_psra_d:
       Opcode = X86ISD::VSRA;
       break;
     }
     return DAG.getNode(Opcode, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   }
 
   // SSE/AVX immediate shift intrinsics
   case Intrinsic::x86_sse2_pslli_w:
   case Intrinsic::x86_sse2_pslli_d:
   case Intrinsic::x86_sse2_pslli_q:
   case Intrinsic::x86_avx2_pslli_w:
   case Intrinsic::x86_avx2_pslli_d:
   case Intrinsic::x86_avx2_pslli_q:
   case Intrinsic::x86_sse2_psrli_w:
   case Intrinsic::x86_sse2_psrli_d:
   case Intrinsic::x86_sse2_psrli_q:
   case Intrinsic::x86_avx2_psrli_w:
   case Intrinsic::x86_avx2_psrli_d:
   case Intrinsic::x86_avx2_psrli_q:
   case Intrinsic::x86_sse2_psrai_w:
   case Intrinsic::x86_sse2_psrai_d:
   case Intrinsic::x86_avx2_psrai_w:
   case Intrinsic::x86_avx2_psrai_d: {
     unsigned Opcode;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_sse2_pslli_w:
     case Intrinsic::x86_sse2_pslli_d:
     case Intrinsic::x86_sse2_pslli_q:
     case Intrinsic::x86_avx2_pslli_w:
     case Intrinsic::x86_avx2_pslli_d:
     case Intrinsic::x86_avx2_pslli_q:
       Opcode = X86ISD::VSHLI;
       break;
     case Intrinsic::x86_sse2_psrli_w:
     case Intrinsic::x86_sse2_psrli_d:
     case Intrinsic::x86_sse2_psrli_q:
     case Intrinsic::x86_avx2_psrli_w:
     case Intrinsic::x86_avx2_psrli_d:
     case Intrinsic::x86_avx2_psrli_q:
       Opcode = X86ISD::VSRLI;
       break;
     case Intrinsic::x86_sse2_psrai_w:
     case Intrinsic::x86_sse2_psrai_d:
     case Intrinsic::x86_avx2_psrai_w:
     case Intrinsic::x86_avx2_psrai_d:
       Opcode = X86ISD::VSRAI;
       break;
     }
     return getTargetVShiftNode(Opcode, dl, Op.getSimpleValueType(),
                                Op.getOperand(1), Op.getOperand(2), DAG);
   }
 
   case Intrinsic::x86_sse42_pcmpistria128:
   case Intrinsic::x86_sse42_pcmpestria128:
   case Intrinsic::x86_sse42_pcmpistric128:
   case Intrinsic::x86_sse42_pcmpestric128:
   case Intrinsic::x86_sse42_pcmpistrio128:
   case Intrinsic::x86_sse42_pcmpestrio128:
   case Intrinsic::x86_sse42_pcmpistris128:
   case Intrinsic::x86_sse42_pcmpestris128:
   case Intrinsic::x86_sse42_pcmpistriz128:
   case Intrinsic::x86_sse42_pcmpestriz128: {
     unsigned Opcode;
     unsigned X86CC;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_sse42_pcmpistria128:
       Opcode = X86ISD::PCMPISTRI;
       X86CC = X86::COND_A;
       break;
     case Intrinsic::x86_sse42_pcmpestria128:
       Opcode = X86ISD::PCMPESTRI;
       X86CC = X86::COND_A;
       break;
     case Intrinsic::x86_sse42_pcmpistric128:
       Opcode = X86ISD::PCMPISTRI;
       X86CC = X86::COND_B;
       break;
     case Intrinsic::x86_sse42_pcmpestric128:
       Opcode = X86ISD::PCMPESTRI;
       X86CC = X86::COND_B;
       break;
     case Intrinsic::x86_sse42_pcmpistrio128:
       Opcode = X86ISD::PCMPISTRI;
       X86CC = X86::COND_O;
       break;
     case Intrinsic::x86_sse42_pcmpestrio128:
       Opcode = X86ISD::PCMPESTRI;
       X86CC = X86::COND_O;
       break;
     case Intrinsic::x86_sse42_pcmpistris128:
       Opcode = X86ISD::PCMPISTRI;
       X86CC = X86::COND_S;
       break;
     case Intrinsic::x86_sse42_pcmpestris128:
       Opcode = X86ISD::PCMPESTRI;
       X86CC = X86::COND_S;
       break;
     case Intrinsic::x86_sse42_pcmpistriz128:
       Opcode = X86ISD::PCMPISTRI;
       X86CC = X86::COND_E;
       break;
     case Intrinsic::x86_sse42_pcmpestriz128:
       Opcode = X86ISD::PCMPESTRI;
       X86CC = X86::COND_E;
       break;
     }
     SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
     SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps);
     SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
                                 DAG.getConstant(X86CC, MVT::i8),
                                 SDValue(PCMP.getNode(), 1));
     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
   }
 
   case Intrinsic::x86_sse42_pcmpistri128:
   case Intrinsic::x86_sse42_pcmpestri128: {
     unsigned Opcode;
     if (IntNo == Intrinsic::x86_sse42_pcmpistri128)
       Opcode = X86ISD::PCMPISTRI;
     else
       Opcode = X86ISD::PCMPESTRI;
 
     SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
     return DAG.getNode(Opcode, dl, VTs, NewOps);
   }
   case Intrinsic::x86_fma_vfmadd_ps:
   case Intrinsic::x86_fma_vfmadd_pd:
   case Intrinsic::x86_fma_vfmsub_ps:
   case Intrinsic::x86_fma_vfmsub_pd:
   case Intrinsic::x86_fma_vfnmadd_ps:
   case Intrinsic::x86_fma_vfnmadd_pd:
   case Intrinsic::x86_fma_vfnmsub_ps:
   case Intrinsic::x86_fma_vfnmsub_pd:
   case Intrinsic::x86_fma_vfmaddsub_ps:
   case Intrinsic::x86_fma_vfmaddsub_pd:
   case Intrinsic::x86_fma_vfmsubadd_ps:
   case Intrinsic::x86_fma_vfmsubadd_pd:
   case Intrinsic::x86_fma_vfmadd_ps_256:
   case Intrinsic::x86_fma_vfmadd_pd_256:
   case Intrinsic::x86_fma_vfmsub_ps_256:
   case Intrinsic::x86_fma_vfmsub_pd_256:
   case Intrinsic::x86_fma_vfnmadd_ps_256:
   case Intrinsic::x86_fma_vfnmadd_pd_256:
   case Intrinsic::x86_fma_vfnmsub_ps_256:
   case Intrinsic::x86_fma_vfnmsub_pd_256:
   case Intrinsic::x86_fma_vfmaddsub_ps_256:
   case Intrinsic::x86_fma_vfmaddsub_pd_256:
   case Intrinsic::x86_fma_vfmsubadd_ps_256:
   case Intrinsic::x86_fma_vfmsubadd_pd_256:
   case Intrinsic::x86_fma_vfmadd_ps_512:
   case Intrinsic::x86_fma_vfmadd_pd_512:
   case Intrinsic::x86_fma_vfmsub_ps_512:
   case Intrinsic::x86_fma_vfmsub_pd_512:
   case Intrinsic::x86_fma_vfnmadd_ps_512:
   case Intrinsic::x86_fma_vfnmadd_pd_512:
   case Intrinsic::x86_fma_vfnmsub_ps_512:
   case Intrinsic::x86_fma_vfnmsub_pd_512:
   case Intrinsic::x86_fma_vfmaddsub_ps_512:
   case Intrinsic::x86_fma_vfmaddsub_pd_512:
   case Intrinsic::x86_fma_vfmsubadd_ps_512:
   case Intrinsic::x86_fma_vfmsubadd_pd_512: {
     unsigned Opc;
     switch (IntNo) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::x86_fma_vfmadd_ps:
     case Intrinsic::x86_fma_vfmadd_pd:
     case Intrinsic::x86_fma_vfmadd_ps_256:
     case Intrinsic::x86_fma_vfmadd_pd_256:
     case Intrinsic::x86_fma_vfmadd_ps_512:
     case Intrinsic::x86_fma_vfmadd_pd_512:
       Opc = X86ISD::FMADD;
       break;
     case Intrinsic::x86_fma_vfmsub_ps:
     case Intrinsic::x86_fma_vfmsub_pd:
     case Intrinsic::x86_fma_vfmsub_ps_256:
     case Intrinsic::x86_fma_vfmsub_pd_256:
     case Intrinsic::x86_fma_vfmsub_ps_512:
     case Intrinsic::x86_fma_vfmsub_pd_512:
       Opc = X86ISD::FMSUB;
       break;
     case Intrinsic::x86_fma_vfnmadd_ps:
     case Intrinsic::x86_fma_vfnmadd_pd:
     case Intrinsic::x86_fma_vfnmadd_ps_256:
     case Intrinsic::x86_fma_vfnmadd_pd_256:
     case Intrinsic::x86_fma_vfnmadd_ps_512:
     case Intrinsic::x86_fma_vfnmadd_pd_512:
       Opc = X86ISD::FNMADD;
       break;
     case Intrinsic::x86_fma_vfnmsub_ps:
     case Intrinsic::x86_fma_vfnmsub_pd:
     case Intrinsic::x86_fma_vfnmsub_ps_256:
     case Intrinsic::x86_fma_vfnmsub_pd_256:
     case Intrinsic::x86_fma_vfnmsub_ps_512:
     case Intrinsic::x86_fma_vfnmsub_pd_512:
       Opc = X86ISD::FNMSUB;
       break;
     case Intrinsic::x86_fma_vfmaddsub_ps:
     case Intrinsic::x86_fma_vfmaddsub_pd:
     case Intrinsic::x86_fma_vfmaddsub_ps_256:
     case Intrinsic::x86_fma_vfmaddsub_pd_256:
     case Intrinsic::x86_fma_vfmaddsub_ps_512:
     case Intrinsic::x86_fma_vfmaddsub_pd_512:
       Opc = X86ISD::FMADDSUB;
       break;
     case Intrinsic::x86_fma_vfmsubadd_ps:
     case Intrinsic::x86_fma_vfmsubadd_pd:
     case Intrinsic::x86_fma_vfmsubadd_ps_256:
     case Intrinsic::x86_fma_vfmsubadd_pd_256:
     case Intrinsic::x86_fma_vfmsubadd_ps_512:
     case Intrinsic::x86_fma_vfmsubadd_pd_512:
       Opc = X86ISD::FMSUBADD;
       break;
     }
 
     return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
                        Op.getOperand(2), Op.getOperand(3));
   }
   }
 }

static SDValue LowerIntVSETCC_AVX512	(	SDValue	Op,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 12100 of file X86ISelLowering.cpp.

                                                                     {
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   SDValue CC = Op.getOperand(2);
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
 
   assert(Op0.getValueType().getVectorElementType().getSizeInBits() >= 32 &&
          Op.getValueType().getScalarType() == MVT::i1 &&
          "Cannot set masked compare for this operation");
 
   ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
   unsigned  Opc = 0;
   bool Unsigned = false;
   bool Swap = false;
   unsigned SSECC;
   switch (SetCCOpcode) {
   default: llvm_unreachable("Unexpected SETCC condition");
   case ISD::SETNE:  SSECC = 4; break;
   case ISD::SETEQ:  Opc = X86ISD::PCMPEQM; break;
   case ISD::SETUGT: SSECC = 6; Unsigned = true; break;
   case ISD::SETLT:  Swap = true; //fall-through
   case ISD::SETGT:  Opc = X86ISD::PCMPGTM; break;
   case ISD::SETULT: SSECC = 1; Unsigned = true; break;
   case ISD::SETUGE: SSECC = 5; Unsigned = true; break; //NLT
   case ISD::SETGE:  Swap = true; SSECC = 2; break; // LE + swap
   case ISD::SETULE: Unsigned = true; //fall-through
   case ISD::SETLE:  SSECC = 2; break;
   }
 
   if (Swap)
     std::swap(Op0, Op1);
   if (Opc)
     return DAG.getNode(Opc, dl, VT, Op0, Op1);
   Opc = Unsigned ? X86ISD::CMPMU: X86ISD::CMPM;
   return DAG.getNode(Opc, dl, VT, Op0, Op1,
                      DAG.getConstant(SSECC, MVT::i8));
 }

static SDValue LowerLOAD_SUB	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 16088 of file X86ISelLowering.cpp.

                                                             {
   SDNode *Node = Op.getNode();
   SDLoc dl(Node);
   EVT T = Node->getValueType(0);
   SDValue negOp = DAG.getNode(ISD::SUB, dl, T,
                               DAG.getConstant(0, T), Node->getOperand(2));
   return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl,
                        cast<AtomicSDNode>(Node)->getMemoryVT(),
                        Node->getOperand(0),
                        Node->getOperand(1), negOp,
                        cast<AtomicSDNode>(Node)->getMemOperand(),
                        cast<AtomicSDNode>(Node)->getOrdering(),
                        cast<AtomicSDNode>(Node)->getSynchScope());
 }

static SDValue LowerMUL	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 15034 of file X86ISelLowering.cpp.

                                            {
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
 
   // Decompose 256-bit ops into smaller 128-bit ops.
   if (VT.is256BitVector() && !Subtarget->hasInt256())
     return Lower256IntArith(Op, DAG);
 
   SDValue A = Op.getOperand(0);
   SDValue B = Op.getOperand(1);
 
   // Lower v4i32 mul as 2x shuffle, 2x pmuludq, 2x shuffle.
   if (VT == MVT::v4i32) {
     assert(Subtarget->hasSSE2() && !Subtarget->hasSSE41() &&
            "Should not custom lower when pmuldq is available!");
 
     // Extract the odd parts.
     static const int UnpackMask[] = { 1, -1, 3, -1 };
     SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
     SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
 
     // Multiply the even parts.
     SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, A, B);
     // Now multiply odd parts.
     SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds);
 
     Evens = DAG.getNode(ISD::BITCAST, dl, VT, Evens);
     Odds = DAG.getNode(ISD::BITCAST, dl, VT, Odds);
 
     // Merge the two vectors back together with a shuffle. This expands into 2
     // shuffles.
     static const int ShufMask[] = { 0, 4, 2, 6 };
     return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
   }
 
   assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
          "Only know how to lower V2I64/V4I64/V8I64 multiply");
 
   //  Ahi = psrlqi(a, 32);
   //  Bhi = psrlqi(b, 32);
   //
   //  AloBlo = pmuludq(a, b);
   //  AloBhi = pmuludq(a, Bhi);
   //  AhiBlo = pmuludq(Ahi, b);
 
   //  AloBhi = psllqi(AloBhi, 32);
   //  AhiBlo = psllqi(AhiBlo, 32);
   //  return AloBlo + AloBhi + AhiBlo;
 
   SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
   SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
 
   // Bit cast to 32-bit vectors for MULUDQ
   EVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 :
                                   (VT == MVT::v4i64) ? MVT::v8i32 : MVT::v16i32;
   A = DAG.getNode(ISD::BITCAST, dl, MulVT, A);
   B = DAG.getNode(ISD::BITCAST, dl, MulVT, B);
   Ahi = DAG.getNode(ISD::BITCAST, dl, MulVT, Ahi);
   Bhi = DAG.getNode(ISD::BITCAST, dl, MulVT, Bhi);
 
   SDValue AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
   SDValue AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
   SDValue AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
 
   AloBhi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AloBhi, 32, DAG);
   AhiBlo = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AhiBlo, 32, DAG);
 
   SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi);
   return DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo);
 }

static SDValue LowerMUL_LOHI	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 15158 of file X86ISelLowering.cpp.

                                                 {
   SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
   EVT VT = Op0.getValueType();
   SDLoc dl(Op);
 
   assert((VT == MVT::v4i32 && Subtarget->hasSSE2()) ||
          (VT == MVT::v8i32 && Subtarget->hasInt256()));
 
   // PMULxD operations multiply each even value (starting at 0) of LHS with
   // the related value of RHS and produce a widen result.
   // E.g., PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
   // => <2 x i64> <ae|cg>
   //
   // In other word, to have all the results, we need to perform two PMULxD:
   // 1. one with the even values.
   // 2. one with the odd values.
   // To achieve #2, with need to place the odd values at an even position.
   //
   // Place the odd value at an even position (basically, shift all values 1
   // step to the left):
   const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1};
   // <a|b|c|d> => <b|undef|d|undef>
   SDValue Odd0 = DAG.getVectorShuffle(VT, dl, Op0, Op0, Mask);
   // <e|f|g|h> => <f|undef|h|undef>
   SDValue Odd1 = DAG.getVectorShuffle(VT, dl, Op1, Op1, Mask);
 
   // Emit two multiplies, one for the lower 2 ints and one for the higher 2
   // ints.
   MVT MulVT = VT == MVT::v4i32 ? MVT::v2i64 : MVT::v4i64;
   bool IsSigned = Op->getOpcode() == ISD::SMUL_LOHI;
   unsigned Opcode =
       (!IsSigned || !Subtarget->hasSSE41()) ? X86ISD::PMULUDQ : X86ISD::PMULDQ;
   // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
   // => <2 x i64> <ae|cg>
   SDValue Mul1 = DAG.getNode(ISD::BITCAST, dl, VT,
                              DAG.getNode(Opcode, dl, MulVT, Op0, Op1));
   // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
   // => <2 x i64> <bf|dh>
   SDValue Mul2 = DAG.getNode(ISD::BITCAST, dl, VT,
                              DAG.getNode(Opcode, dl, MulVT, Odd0, Odd1));
 
   // Shuffle it back into the right order.
   // The internal representation is big endian.
   // In other words, a i64 bitcasted to 2 x i32 has its high part at index 0
   // and its low part at index 1.
   // Moreover, we have: Mul1 = <ae|cg> ; Mul2 = <bf|dh>
   // Vector index                0 1   ;          2 3
   // We want      <ae|bf|cg|dh>
   // Vector index   0  2  1  3
   // Since each element is seen as 2 x i32, we get:
   // high_mask[i] = 2 x vector_index[i]
   // low_mask[i] = 2 x vector_index[i] + 1
   // where vector_index = {0, Size/2, 1, Size/2 + 1, ...,
   //                       Size/2 - 1, Size/2 + Size/2 - 1}
   // where Size is the number of element of the final vector.
   SDValue Highs, Lows;
   if (VT == MVT::v8i32) {
     const int HighMask[] = {0, 8, 2, 10, 4, 12, 6, 14};
     Highs = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, HighMask);
     const int LowMask[] = {1, 9, 3, 11, 5, 13, 7, 15};
     Lows = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, LowMask);
   } else {
     const int HighMask[] = {0, 4, 2, 6};
     Highs = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, HighMask);
     const int LowMask[] = {1, 5, 3, 7};
     Lows = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, LowMask);
   }
 
   // If we have a signed multiply but no PMULDQ fix up the high parts of a
   // unsigned multiply.
   if (IsSigned && !Subtarget->hasSSE41()) {
     SDValue ShAmt =
         DAG.getConstant(31, DAG.getTargetLoweringInfo().getShiftAmountTy(VT));
     SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
                              DAG.getNode(ISD::SRA, dl, VT, Op0, ShAmt), Op1);
     SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
                              DAG.getNode(ISD::SRA, dl, VT, Op1, ShAmt), Op0);
 
     SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
     Highs = DAG.getNode(ISD::SUB, dl, VT, Highs, Fixup);
   }
 
   // The low part of a MUL_LOHI is supposed to be the first value and the
   // high part the second value.
   return DAG.getNode(ISD::MERGE_VALUES, dl, Op.getValueType(), Lows, Highs);
 }

static SDValue LowerREADCYCLECOUNTER	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 14376 of file X86ISelLowering.cpp.

                                                         {
   SmallVector<SDValue, 2> Results;
   SDLoc DL(Op);
   getReadTimeStampCounter(Op.getNode(), DL, X86ISD::RDTSC_DAG, DAG, Subtarget,
                           Results);
   return DAG.getMergeValues(Results, DL);
 }

static SDValue LowerSCALAR_TO_VECTOR	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 10056 of file X86ISelLowering.cpp.

                                                                     {
   SDLoc dl(Op);
   MVT OpVT = Op.getSimpleValueType();
 
   // If this is a 256-bit vector result, first insert into a 128-bit
   // vector and then insert into the 256-bit vector.
   if (!OpVT.is128BitVector()) {
     // Insert into a 128-bit vector.
     unsigned SizeFactor = OpVT.getSizeInBits()/128;
     MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(),
                                  OpVT.getVectorNumElements() / SizeFactor);
 
     Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
 
     // Insert the 128-bit vector.
     return Insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
   }
 
   if (OpVT == MVT::v1i64 &&
       Op.getOperand(0).getValueType() == MVT::i64)
     return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0));
 
   SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
   assert(OpVT.is128BitVector() && "Expected an SSE type!");
   return DAG.getNode(ISD::BITCAST, dl, OpVT,
                      DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt));
 }

static SDValue LowerScalarImmediateShift	(	SDValue	Op,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 15246 of file X86ISelLowering.cpp.

                                                                         {
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
   SDValue R = Op.getOperand(0);
   SDValue Amt = Op.getOperand(1);
 
   // Optimize shl/srl/sra with constant shift amount.
   if (auto *BVAmt = dyn_cast<BuildVectorSDNode>(Amt)) {
     if (auto *ShiftConst = BVAmt->getConstantSplatNode()) {
       uint64_t ShiftAmt = ShiftConst->getZExtValue();
 
       if (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
           (Subtarget->hasInt256() &&
            (VT == MVT::v4i64 || VT == MVT::v8i32 || VT == MVT::v16i16)) ||
           (Subtarget->hasAVX512() &&
            (VT == MVT::v8i64 || VT == MVT::v16i32))) {
         if (Op.getOpcode() == ISD::SHL)
           return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
                                             DAG);
         if (Op.getOpcode() == ISD::SRL)
           return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
                                             DAG);
         if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64)
           return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
                                             DAG);
       }
 
       if (VT == MVT::v16i8) {
         if (Op.getOpcode() == ISD::SHL) {
           // Make a large shift.
           SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl,
                                                    MVT::v8i16, R, ShiftAmt,
                                                    DAG);
           SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
           // Zero out the rightmost bits.
           SmallVector<SDValue, 16> V(16,
                                      DAG.getConstant(uint8_t(-1U << ShiftAmt),
                                                      MVT::i8));
           return DAG.getNode(ISD::AND, dl, VT, SHL,
                              DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
         }
         if (Op.getOpcode() == ISD::SRL) {
           // Make a large shift.
           SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl,
                                                    MVT::v8i16, R, ShiftAmt,
                                                    DAG);
           SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
           // Zero out the leftmost bits.
           SmallVector<SDValue, 16> V(16,
                                      DAG.getConstant(uint8_t(-1U) >> ShiftAmt,
                                                      MVT::i8));
           return DAG.getNode(ISD::AND, dl, VT, SRL,
                              DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
         }
         if (Op.getOpcode() == ISD::SRA) {
           if (ShiftAmt == 7) {
             // R s>> 7  ===  R s< 0
             SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
             return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
           }
 
           // R s>> a === ((R u>> a) ^ m) - m
           SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
           SmallVector<SDValue, 16> V(16, DAG.getConstant(128 >> ShiftAmt,
                                                          MVT::i8));
           SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V);
           Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
           Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
           return Res;
         }
         llvm_unreachable("Unknown shift opcode.");
       }
 
       if (Subtarget->hasInt256() && VT == MVT::v32i8) {
         if (Op.getOpcode() == ISD::SHL) {
           // Make a large shift.
           SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl,
                                                    MVT::v16i16, R, ShiftAmt,
                                                    DAG);
           SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
           // Zero out the rightmost bits.
           SmallVector<SDValue, 32> V(32,
                                      DAG.getConstant(uint8_t(-1U << ShiftAmt),
                                                      MVT::i8));
           return DAG.getNode(ISD::AND, dl, VT, SHL,
                              DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
         }
         if (Op.getOpcode() == ISD::SRL) {
           // Make a large shift.
           SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl,
                                                    MVT::v16i16, R, ShiftAmt,
                                                    DAG);
           SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
           // Zero out the leftmost bits.
           SmallVector<SDValue, 32> V(32,
                                      DAG.getConstant(uint8_t(-1U) >> ShiftAmt,
                                                      MVT::i8));
           return DAG.getNode(ISD::AND, dl, VT, SRL,
                              DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
         }
         if (Op.getOpcode() == ISD::SRA) {
           if (ShiftAmt == 7) {
             // R s>> 7  ===  R s< 0
             SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
             return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
           }
 
           // R s>> a === ((R u>> a) ^ m) - m
           SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
           SmallVector<SDValue, 32> V(32, DAG.getConstant(128 >> ShiftAmt,
                                                          MVT::i8));
           SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V);
           Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
           Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
           return Res;
         }
         llvm_unreachable("Unknown shift opcode.");
       }
     }
   }
 
   // Special case in 32-bit mode, where i64 is expanded into high and low parts.
   if (!Subtarget->is64Bit() &&
       (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64)) &&
       Amt.getOpcode() == ISD::BITCAST &&
       Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
     Amt = Amt.getOperand(0);
     unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() /
                      VT.getVectorNumElements();
     unsigned RatioInLog2 = Log2_32_Ceil(Ratio);
     uint64_t ShiftAmt = 0;
     for (unsigned i = 0; i != Ratio; ++i) {
       ConstantSDNode *C = dyn_cast<ConstantSDNode>(Amt.getOperand(i));
       if (!C)
         return SDValue();
       // 6 == Log2(64)
       ShiftAmt |= C->getZExtValue() << (i * (1 << (6 - RatioInLog2)));
     }
     // Check remaining shift amounts.
     for (unsigned i = Ratio; i != Amt.getNumOperands(); i += Ratio) {
       uint64_t ShAmt = 0;
       for (unsigned j = 0; j != Ratio; ++j) {
         ConstantSDNode *C =
           dyn_cast<ConstantSDNode>(Amt.getOperand(i + j));
         if (!C)
           return SDValue();
         // 6 == Log2(64)
         ShAmt |= C->getZExtValue() << (j * (1 << (6 - RatioInLog2)));
       }
       if (ShAmt != ShiftAmt)
         return SDValue();
     }
     switch (Op.getOpcode()) {
     default:
       llvm_unreachable("Unknown shift opcode!");
     case ISD::SHL:
       return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
                                         DAG);
     case ISD::SRL:
       return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
                                         DAG);
     case ISD::SRA:
       return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
                                         DAG);
     }
   }
 
   return SDValue();
 }

static SDValue LowerScalarVariableShift	(	SDValue	Op,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 15417 of file X86ISelLowering.cpp.

                                                                        {
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
   SDValue R = Op.getOperand(0);
   SDValue Amt = Op.getOperand(1);
 
   if ((VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) ||
       VT == MVT::v4i32 || VT == MVT::v8i16 ||
       (Subtarget->hasInt256() &&
        ((VT == MVT::v4i64 && Op.getOpcode() != ISD::SRA) ||
         VT == MVT::v8i32 || VT == MVT::v16i16)) ||
        (Subtarget->hasAVX512() && (VT == MVT::v8i64 || VT == MVT::v16i32))) {
     SDValue BaseShAmt;
     EVT EltVT = VT.getVectorElementType();
 
     if (Amt.getOpcode() == ISD::BUILD_VECTOR) {
       unsigned NumElts = VT.getVectorNumElements();
       unsigned i, j;
       for (i = 0; i != NumElts; ++i) {
         if (Amt.getOperand(i).getOpcode() == ISD::UNDEF)
           continue;
         break;
       }
       for (j = i; j != NumElts; ++j) {
         SDValue Arg = Amt.getOperand(j);
         if (Arg.getOpcode() == ISD::UNDEF) continue;
         if (Arg != Amt.getOperand(i))
           break;
       }
       if (i != NumElts && j == NumElts)
         BaseShAmt = Amt.getOperand(i);
     } else {
       if (Amt.getOpcode() == ISD::EXTRACT_SUBVECTOR)
         Amt = Amt.getOperand(0);
       if (Amt.getOpcode() == ISD::VECTOR_SHUFFLE &&
                cast<ShuffleVectorSDNode>(Amt)->isSplat()) {
         SDValue InVec = Amt.getOperand(0);
         if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
           unsigned NumElts = InVec.getValueType().getVectorNumElements();
           unsigned i = 0;
           for (; i != NumElts; ++i) {
             SDValue Arg = InVec.getOperand(i);
             if (Arg.getOpcode() == ISD::UNDEF) continue;
             BaseShAmt = Arg;
             break;
           }
         } else if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT) {
            if (ConstantSDNode *C =
                dyn_cast<ConstantSDNode>(InVec.getOperand(2))) {
              unsigned SplatIdx =
                cast<ShuffleVectorSDNode>(Amt)->getSplatIndex();
              if (C->getZExtValue() == SplatIdx)
                BaseShAmt = InVec.getOperand(1);
            }
         }
         if (!BaseShAmt.getNode())
           BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Amt,
                                   DAG.getIntPtrConstant(0));
       }
     }
 
     if (BaseShAmt.getNode()) {
       if (EltVT.bitsGT(MVT::i32))
         BaseShAmt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BaseShAmt);
       else if (EltVT.bitsLT(MVT::i32))
         BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt);
 
       switch (Op.getOpcode()) {
       default:
         llvm_unreachable("Unknown shift opcode!");
       case ISD::SHL:
         switch (VT.SimpleTy) {
         default: return SDValue();
         case MVT::v2i64:
         case MVT::v4i32:
         case MVT::v8i16:
         case MVT::v4i64:
         case MVT::v8i32:
         case MVT::v16i16:
         case MVT::v16i32:
         case MVT::v8i64:
           return getTargetVShiftNode(X86ISD::VSHLI, dl, VT, R, BaseShAmt, DAG);
         }
       case ISD::SRA:
         switch (VT.SimpleTy) {
         default: return SDValue();
         case MVT::v4i32:
         case MVT::v8i16:
         case MVT::v8i32:
         case MVT::v16i16:
         case MVT::v16i32:
         case MVT::v8i64:
           return getTargetVShiftNode(X86ISD::VSRAI, dl, VT, R, BaseShAmt, DAG);
         }
       case ISD::SRL:
         switch (VT.SimpleTy) {
         default: return SDValue();
         case MVT::v2i64:
         case MVT::v4i32:
         case MVT::v8i16:
         case MVT::v4i64:
         case MVT::v8i32:
         case MVT::v16i16:
         case MVT::v16i32:
         case MVT::v8i64:
           return getTargetVShiftNode(X86ISD::VSRLI, dl, VT, R, BaseShAmt, DAG);
         }
       }
     }
   }
 
   // Special case in 32-bit mode, where i64 is expanded into high and low parts.
   if (!Subtarget->is64Bit() &&
       (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64) ||
       (Subtarget->hasAVX512() && VT == MVT::v8i64)) &&
       Amt.getOpcode() == ISD::BITCAST &&
       Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
     Amt = Amt.getOperand(0);
     unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() /
                      VT.getVectorNumElements();
     std::vector<SDValue> Vals(Ratio);
     for (unsigned i = 0; i != Ratio; ++i)
       Vals[i] = Amt.getOperand(i);
     for (unsigned i = Ratio; i != Amt.getNumOperands(); i += Ratio) {
       for (unsigned j = 0; j != Ratio; ++j)
         if (Vals[j] != Amt.getOperand(i + j))
           return SDValue();
     }
     switch (Op.getOpcode()) {
     default:
       llvm_unreachable("Unknown shift opcode!");
     case ISD::SHL:
       return DAG.getNode(X86ISD::VSHL, dl, VT, R, Op.getOperand(1));
     case ISD::SRL:
       return DAG.getNode(X86ISD::VSRL, dl, VT, R, Op.getOperand(1));
     case ISD::SRA:
       return DAG.getNode(X86ISD::VSRA, dl, VT, R, Op.getOperand(1));
     }
   }
 
   return SDValue();
 }

static SDValue LowerShift	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 15561 of file X86ISelLowering.cpp.

                                              {
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
   SDValue R = Op.getOperand(0);
   SDValue Amt = Op.getOperand(1);
   SDValue V;
 
   assert(VT.isVector() && "Custom lowering only for vector shifts!");
   assert(Subtarget->hasSSE2() && "Only custom lower when we have SSE2!");
 
   V = LowerScalarImmediateShift(Op, DAG, Subtarget);
   if (V.getNode())
     return V;
 
   V = LowerScalarVariableShift(Op, DAG, Subtarget);
   if (V.getNode())
       return V;
 
   if (Subtarget->hasAVX512() && (VT == MVT::v16i32 || VT == MVT::v8i64))
     return Op;
   // AVX2 has VPSLLV/VPSRAV/VPSRLV.
   if (Subtarget->hasInt256()) {
     if (Op.getOpcode() == ISD::SRL &&
         (VT == MVT::v2i64 || VT == MVT::v4i32 ||
          VT == MVT::v4i64 || VT == MVT::v8i32))
       return Op;
     if (Op.getOpcode() == ISD::SHL &&
         (VT == MVT::v2i64 || VT == MVT::v4i32 ||
          VT == MVT::v4i64 || VT == MVT::v8i32))
       return Op;
     if (Op.getOpcode() == ISD::SRA && (VT == MVT::v4i32 || VT == MVT::v8i32))
       return Op;
   }
 
   // If possible, lower this packed shift into a vector multiply instead of
   // expanding it into a sequence of scalar shifts.
   // Do this only if the vector shift count is a constant build_vector.
   if (Op.getOpcode() == ISD::SHL && 
       (VT == MVT::v8i16 || VT == MVT::v4i32 ||
        (Subtarget->hasInt256() && VT == MVT::v16i16)) &&
       ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
     SmallVector<SDValue, 8> Elts;
     EVT SVT = VT.getScalarType();
     unsigned SVTBits = SVT.getSizeInBits();
     const APInt &One = APInt(SVTBits, 1);
     unsigned NumElems = VT.getVectorNumElements();
 
     for (unsigned i=0; i !=NumElems; ++i) {
       SDValue Op = Amt->getOperand(i);
       if (Op->getOpcode() == ISD::UNDEF) {
         Elts.push_back(Op);
         continue;
       }
 
       ConstantSDNode *ND = cast<ConstantSDNode>(Op);
       const APInt &C = APInt(SVTBits, ND->getAPIntValue().getZExtValue());
       uint64_t ShAmt = C.getZExtValue();
       if (ShAmt >= SVTBits) {
         Elts.push_back(DAG.getUNDEF(SVT));
         continue;
       }
       Elts.push_back(DAG.getConstant(One.shl(ShAmt), SVT));
     }
     SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts);
     return DAG.getNode(ISD::MUL, dl, VT, R, BV);
   }
 
   // Lower SHL with variable shift amount.
   if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
     Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, VT));
 
     Op = DAG.getNode(ISD::ADD, dl, VT, Op, DAG.getConstant(0x3f800000U, VT));
     Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op);
     Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
     return DAG.getNode(ISD::MUL, dl, VT, Op, R);
   }
 
   // If possible, lower this shift as a sequence of two shifts by
   // constant plus a MOVSS/MOVSD instead of scalarizing it.
   // Example:
   //   (v4i32 (srl A, (build_vector < X, Y, Y, Y>)))
   //
   // Could be rewritten as:
   //   (v4i32 (MOVSS (srl A, <Y,Y,Y,Y>), (srl A, <X,X,X,X>)))
   //
   // The advantage is that the two shifts from the example would be
   // lowered as X86ISD::VSRLI nodes. This would be cheaper than scalarizing
   // the vector shift into four scalar shifts plus four pairs of vector
   // insert/extract.
   if ((VT == MVT::v8i16 || VT == MVT::v4i32) &&
       ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
     unsigned TargetOpcode = X86ISD::MOVSS;
     bool CanBeSimplified;
     // The splat value for the first packed shift (the 'X' from the example).
     SDValue Amt1 = Amt->getOperand(0);
     // The splat value for the second packed shift (the 'Y' from the example).
     SDValue Amt2 = (VT == MVT::v4i32) ? Amt->getOperand(1) :
                                         Amt->getOperand(2);
 
     // See if it is possible to replace this node with a sequence of
     // two shifts followed by a MOVSS/MOVSD
     if (VT == MVT::v4i32) {
       // Check if it is legal to use a MOVSS.
       CanBeSimplified = Amt2 == Amt->getOperand(2) &&
                         Amt2 == Amt->getOperand(3);
       if (!CanBeSimplified) {
         // Otherwise, check if we can still simplify this node using a MOVSD.
         CanBeSimplified = Amt1 == Amt->getOperand(1) &&
                           Amt->getOperand(2) == Amt->getOperand(3);
         TargetOpcode = X86ISD::MOVSD;
         Amt2 = Amt->getOperand(2);
       }
     } else {
       // Do similar checks for the case where the machine value type
       // is MVT::v8i16.
       CanBeSimplified = Amt1 == Amt->getOperand(1);
       for (unsigned i=3; i != 8 && CanBeSimplified; ++i)
         CanBeSimplified = Amt2 == Amt->getOperand(i);
 
       if (!CanBeSimplified) {
         TargetOpcode = X86ISD::MOVSD;
         CanBeSimplified = true;
         Amt2 = Amt->getOperand(4);
         for (unsigned i=0; i != 4 && CanBeSimplified; ++i)
           CanBeSimplified = Amt1 == Amt->getOperand(i);
         for (unsigned j=4; j != 8 && CanBeSimplified; ++j)
           CanBeSimplified = Amt2 == Amt->getOperand(j);
       }
     }
     
     if (CanBeSimplified && isa<ConstantSDNode>(Amt1) &&
         isa<ConstantSDNode>(Amt2)) {
       // Replace this node with two shifts followed by a MOVSS/MOVSD.
       EVT CastVT = MVT::v4i32;
       SDValue Splat1 = 
         DAG.getConstant(cast<ConstantSDNode>(Amt1)->getAPIntValue(), VT);
       SDValue Shift1 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat1);
       SDValue Splat2 = 
         DAG.getConstant(cast<ConstantSDNode>(Amt2)->getAPIntValue(), VT);
       SDValue Shift2 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat2);
       if (TargetOpcode == X86ISD::MOVSD)
         CastVT = MVT::v2i64;
       SDValue BitCast1 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift1);
       SDValue BitCast2 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift2);
       SDValue Result = getTargetShuffleNode(TargetOpcode, dl, CastVT, BitCast2,
                                             BitCast1, DAG);
       return DAG.getNode(ISD::BITCAST, dl, VT, Result);
     }
   }
 
   if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) {
     assert(Subtarget->hasSSE2() && "Need SSE2 for pslli/pcmpeq.");
 
     // a = a << 5;
     Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(5, VT));
     Op = DAG.getNode(ISD::BITCAST, dl, VT, Op);
 
     // Turn 'a' into a mask suitable for VSELECT
     SDValue VSelM = DAG.getConstant(0x80, VT);
     SDValue OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
     OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
 
     SDValue CM1 = DAG.getConstant(0x0f, VT);
     SDValue CM2 = DAG.getConstant(0x3f, VT);
 
     // r = VSELECT(r, psllw(r & (char16)15, 4), a);
     SDValue M = DAG.getNode(ISD::AND, dl, VT, R, CM1);
     M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 4, DAG);
     M = DAG.getNode(ISD::BITCAST, dl, VT, M);
     R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
 
     // a += a
     Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
     OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
     OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
 
     // r = VSELECT(r, psllw(r & (char16)63, 2), a);
     M = DAG.getNode(ISD::AND, dl, VT, R, CM2);
     M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 2, DAG);
     M = DAG.getNode(ISD::BITCAST, dl, VT, M);
     R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
 
     // a += a
     Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
     OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
     OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
 
     // return VSELECT(r, r+r, a);
     R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel,
                     DAG.getNode(ISD::ADD, dl, VT, R, R), R);
     return R;
   }
 
   // It's worth extending once and using the v8i32 shifts for 16-bit types, but
   // the extra overheads to get from v16i8 to v8i32 make the existing SSE
   // solution better.
   if (Subtarget->hasInt256() && VT == MVT::v8i16) {
     MVT NewVT = VT == MVT::v8i16 ? MVT::v8i32 : MVT::v16i16;
     unsigned ExtOpc =
         Op.getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
     R = DAG.getNode(ExtOpc, dl, NewVT, R);
     Amt = DAG.getNode(ISD::ANY_EXTEND, dl, NewVT, Amt);
     return DAG.getNode(ISD::TRUNCATE, dl, VT,
                        DAG.getNode(Op.getOpcode(), dl, NewVT, R, Amt));
     }
 
   // Decompose 256-bit shifts into smaller 128-bit shifts.
   if (VT.is256BitVector()) {
     unsigned NumElems = VT.getVectorNumElements();
     MVT EltVT = VT.getVectorElementType();
     EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
 
     // Extract the two vectors
     SDValue V1 = Extract128BitVector(R, 0, DAG, dl);
     SDValue V2 = Extract128BitVector(R, NumElems/2, DAG, dl);
 
     // Recreate the shift amount vectors
     SDValue Amt1, Amt2;
     if (Amt.getOpcode() == ISD::BUILD_VECTOR) {
       // Constant shift amount
       SmallVector<SDValue, 4> Amt1Csts;
       SmallVector<SDValue, 4> Amt2Csts;
       for (unsigned i = 0; i != NumElems/2; ++i)
         Amt1Csts.push_back(Amt->getOperand(i));
       for (unsigned i = NumElems/2; i != NumElems; ++i)
         Amt2Csts.push_back(Amt->getOperand(i));
 
       Amt1 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Amt1Csts);
       Amt2 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Amt2Csts);
     } else {
       // Variable shift amount
       Amt1 = Extract128BitVector(Amt, 0, DAG, dl);
       Amt2 = Extract128BitVector(Amt, NumElems/2, DAG, dl);
     }
 
     // Issue new vector shifts for the smaller types
     V1 = DAG.getNode(Op.getOpcode(), dl, NewVT, V1, Amt1);
     V2 = DAG.getNode(Op.getOpcode(), dl, NewVT, V2, Amt2);
 
     // Concatenate the result back
     return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, V1, V2);
   }
 
   return SDValue();
 }

static SDValue LowerShiftParts	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.

Definition at line 10625 of file X86ISelLowering.cpp.

                                                               {
   assert(Op.getNumOperands() == 3 && "Not a double-shift!");
   MVT VT = Op.getSimpleValueType();
   unsigned VTBits = VT.getSizeInBits();
   SDLoc dl(Op);
   bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
   SDValue ShOpLo = Op.getOperand(0);
   SDValue ShOpHi = Op.getOperand(1);
   SDValue ShAmt  = Op.getOperand(2);
   // X86ISD::SHLD and X86ISD::SHRD have defined overflow behavior but the
   // generic ISD nodes haven't. Insert an AND to be safe, it's optimized away
   // during isel.
   SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
                                   DAG.getConstant(VTBits - 1, MVT::i8));
   SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
                                      DAG.getConstant(VTBits - 1, MVT::i8))
                        : DAG.getConstant(0, VT);
 
   SDValue Tmp2, Tmp3;
   if (Op.getOpcode() == ISD::SHL_PARTS) {
     Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
   } else {
     Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
     Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
   }
 
   // If the shift amount is larger or equal than the width of a part we can't
   // rely on the results of shld/shrd. Insert a test and select the appropriate
   // values for large shift amounts.
   SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
                                 DAG.getConstant(VTBits, MVT::i8));
   SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
                              AndNode, DAG.getConstant(0, MVT::i8));
 
   SDValue Hi, Lo;
   SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8);
   SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond };
   SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond };
 
   if (Op.getOpcode() == ISD::SHL_PARTS) {
     Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
     Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
   } else {
     Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
     Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
   }
 
   SDValue Ops[2] = { Lo, Hi };
   return DAG.getMergeValues(Ops, dl);
 }

static SDValue LowerSIGN_EXTEND	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 12763 of file X86ISelLowering.cpp.

                                                    {
   MVT VT = Op->getSimpleValueType(0);
   SDValue In = Op->getOperand(0);
   MVT InVT = In.getSimpleValueType();
   SDLoc dl(Op);
 
   if (VT.is512BitVector() || InVT.getVectorElementType() == MVT::i1)
     return LowerSIGN_EXTEND_AVX512(Op, DAG);
 
   if ((VT != MVT::v4i64 || InVT != MVT::v4i32) &&
       (VT != MVT::v8i32 || InVT != MVT::v8i16) &&
       (VT != MVT::v16i16 || InVT != MVT::v16i8))
     return SDValue();
 
   if (Subtarget->hasInt256())
     return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
 
   // Optimize vectors in AVX mode
   // Sign extend  v8i16 to v8i32 and
   //              v4i32 to v4i64
   //
   // Divide input vector into two parts
   // for v4i32 the shuffle mask will be { 0, 1, -1, -1} {2, 3, -1, -1}
   // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32
   // concat the vectors to original VT
 
   unsigned NumElems = InVT.getVectorNumElements();
   SDValue Undef = DAG.getUNDEF(InVT);
 
   SmallVector<int,8> ShufMask1(NumElems, -1);
   for (unsigned i = 0; i != NumElems/2; ++i)
     ShufMask1[i] = i;
 
   SDValue OpLo = DAG.getVectorShuffle(InVT, dl, In, Undef, &ShufMask1[0]);
 
   SmallVector<int,8> ShufMask2(NumElems, -1);
   for (unsigned i = 0; i != NumElems/2; ++i)
     ShufMask2[i] = i + NumElems/2;
 
   SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, Undef, &ShufMask2[0]);
 
   MVT HalfVT = MVT::getVectorVT(VT.getScalarType(),
                                 VT.getVectorNumElements()/2);
 
   OpLo = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpLo);
   OpHi = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpHi);
 
   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
 }

static SDValue LowerSIGN_EXTEND_AVX512	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 12732 of file X86ISelLowering.cpp.

                                                                       {
   MVT VT = Op->getSimpleValueType(0);
   SDValue In = Op->getOperand(0);
   MVT InVT = In.getSimpleValueType();
   SDLoc dl(Op);
 
   unsigned int NumElts = VT.getVectorNumElements();
   if (NumElts != 8 && NumElts != 16)
     return SDValue();
 
   if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1)
     return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   assert (InVT.getVectorElementType() == MVT::i1 && "Unexpected vector type");
 
   MVT ExtVT = (NumElts == 8) ? MVT::v8i64 : MVT::v16i32;
   Constant *C = ConstantInt::get(*DAG.getContext(),
     APInt::getAllOnesValue(ExtVT.getScalarType().getSizeInBits()));
 
   SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
   unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
   SDValue Ld = DAG.getLoad(ExtVT.getScalarType(), dl, DAG.getEntryNode(), CP,
                           MachinePointerInfo::getConstantPool(),
                           false, false, false, Alignment);
   SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, dl, ExtVT, In, Ld);
   if (VT.is512BitVector())
     return Brcst;
   return DAG.getNode(X86ISD::VTRUNC, dl, VT, Brcst);
 }

static SDValue LowerSUB	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 15027 of file X86ISelLowering.cpp.

                                                        {
   assert(Op.getSimpleValueType().is256BitVector() &&
          Op.getSimpleValueType().isInteger() &&
          "Only handle AVX 256-bit vector integer operation");
   return Lower256IntArith(Op, DAG);
 }

static SDValue LowerToTLSExecModel	(	GlobalAddressSDNode *	GA,
		SelectionDAG &	DAG,
		const EVT	PtrVT,
		TLSModel::Model	model,
		bool	is64Bit,
		bool	isPIC
	)

static

Definition at line 10430 of file X86ISelLowering.cpp.

                                                              {
   SDLoc dl(GA);
 
   // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
   Value *Ptr = Constant::getNullValue(Type::getInt8PtrTy(*DAG.getContext(),
                                                          is64Bit ? 257 : 256));
 
   SDValue ThreadPointer =
       DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0),
                   MachinePointerInfo(Ptr), false, false, false, 0);
 
   unsigned char OperandFlags = 0;
   // Most TLS accesses are not RIP relative, even on x86-64.  One exception is
   // initialexec.
   unsigned WrapperKind = X86ISD::Wrapper;
   if (model == TLSModel::LocalExec) {
     OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
   } else if (model == TLSModel::InitialExec) {
     if (is64Bit) {
       OperandFlags = X86II::MO_GOTTPOFF;
       WrapperKind = X86ISD::WrapperRIP;
     } else {
       OperandFlags = isPIC ? X86II::MO_GOTNTPOFF : X86II::MO_INDNTPOFF;
     }
   } else {
     llvm_unreachable("Unexpected model");
   }
 
   // emit "addl x@ntpoff,%eax" (local exec)
   // or "addl x@indntpoff,%eax" (initial exec)
   // or "addl x@gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
   SDValue TGA =
       DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
                                  GA->getOffset(), OperandFlags);
   SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
 
   if (model == TLSModel::InitialExec) {
     if (isPIC && !is64Bit) {
       Offset = DAG.getNode(ISD::ADD, dl, PtrVT,
                            DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
                            Offset);
     }
 
     Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
                          MachinePointerInfo::getGOT(), false, false, false, 0);
   }
 
   // The address of the thread local variable is the add of the thread
   // pointer with the offset of the variable.
   return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
 }

static SDValue LowerToTLSGeneralDynamicModel32	(	GlobalAddressSDNode *	GA,
		SelectionDAG &	DAG,
		const EVT	PtrVT
	)

static

Definition at line 10370 of file X86ISelLowering.cpp.

                                                  {
   SDValue InFlag;
   SDLoc dl(GA);  // ? function entry point might be better
   SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
                                    DAG.getNode(X86ISD::GlobalBaseReg,
                                                SDLoc(), PtrVT), InFlag);
   InFlag = Chain.getValue(1);
 
   return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
 }

static SDValue LowerToTLSGeneralDynamicModel64	(	GlobalAddressSDNode *	GA,
		SelectionDAG &	DAG,
		const EVT	PtrVT
	)

static

Definition at line 10384 of file X86ISelLowering.cpp.

                                                  {
   return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
                     X86::RAX, X86II::MO_TLSGD);
 }

static SDValue LowerToTLSLocalDynamicModel	(	GlobalAddressSDNode *	GA,
		SelectionDAG &	DAG,
		const EVT	PtrVT,
		bool	is64Bit
	)

static

Definition at line 10390 of file X86ISelLowering.cpp.

                                                          {
   SDLoc dl(GA);
 
   // Get the start address of the TLS block for this module.
   X86MachineFunctionInfo* MFI = DAG.getMachineFunction()
       .getInfo<X86MachineFunctionInfo>();
   MFI->incNumLocalDynamicTLSAccesses();
 
   SDValue Base;
   if (is64Bit) {
     Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, X86::RAX,
                       X86II::MO_TLSLD, /*LocalDynamic=*/true);
   } else {
     SDValue InFlag;
     SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
         DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InFlag);
     InFlag = Chain.getValue(1);
     Base = GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX,
                       X86II::MO_TLSLDM, /*LocalDynamic=*/true);
   }
 
   // Note: the CleanupLocalDynamicTLSPass will remove redundant computations
   // of Base.
 
   // Build x@dtpoff.
   unsigned char OperandFlags = X86II::MO_DTPOFF;
   unsigned WrapperKind = X86ISD::Wrapper;
   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
                                            GA->getValueType(0),
                                            GA->getOffset(), OperandFlags);
   SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
 
   // Add x@dtpoff with the base.
   return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base);
 }

static SDValue lowerV16I8VectorShuffle	(	SDValue	Op,
		SDValue	V1,
		SDValue	V2,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Generic lowering of v16i8 shuffles.

This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.

Definition at line 7668 of file X86ISelLowering.cpp.

                                                           {
   SDLoc DL(Op);
   assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!");
   assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   ArrayRef<int> OrigMask = SVOp->getMask();
   assert(OrigMask.size() == 16 && "Unexpected mask size for v16 shuffle!");
   int MaskStorage[16] = {
       OrigMask[0],  OrigMask[1],  OrigMask[2],  OrigMask[3],
       OrigMask[4],  OrigMask[5],  OrigMask[6],  OrigMask[7],
       OrigMask[8],  OrigMask[9],  OrigMask[10], OrigMask[11],
       OrigMask[12], OrigMask[13], OrigMask[14], OrigMask[15]};
   MutableArrayRef<int> Mask(MaskStorage);
   MutableArrayRef<int> LoMask = Mask.slice(0, 8);
   MutableArrayRef<int> HiMask = Mask.slice(8, 8);
 
   // For single-input shuffles, there are some nicer lowering tricks we can use.
   if (isSingleInputShuffleMask(Mask)) {
     // Check whether we can widen this to an i16 shuffle by duplicating bytes.
     // Notably, this handles splat and partial-splat shuffles more efficiently.
     // However, it only makes sense if the pre-duplication shuffle simplifies
     // things significantly. Currently, this means we need to be able to
     // express the pre-duplication shuffle as an i16 shuffle.
     //
     // FIXME: We should check for other patterns which can be widened into an
     // i16 shuffle as well.
     auto canWidenViaDuplication = [](ArrayRef<int> Mask) {
       for (int i = 0; i < 16; i += 2) {
         if (Mask[i] != Mask[i + 1])
           return false;
       }
       return true;
     };
     auto tryToWidenViaDuplication = [&]() -> SDValue {
       if (!canWidenViaDuplication(Mask))
         return SDValue();
       SmallVector<int, 4> LoInputs;
       std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(LoInputs),
                    [](int M) { return M >= 0 && M < 8; });
       std::sort(LoInputs.begin(), LoInputs.end());
       LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()),
                      LoInputs.end());
       SmallVector<int, 4> HiInputs;
       std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(HiInputs),
                    [](int M) { return M >= 8; });
       std::sort(HiInputs.begin(), HiInputs.end());
       HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()),
                      HiInputs.end());
 
       bool TargetLo = LoInputs.size() >= HiInputs.size();
       ArrayRef<int> InPlaceInputs = TargetLo ? LoInputs : HiInputs;
       ArrayRef<int> MovingInputs = TargetLo ? HiInputs : LoInputs;
 
       int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
       SmallDenseMap<int, int, 8> LaneMap;
       for (int I : InPlaceInputs) {
         PreDupI16Shuffle[I/2] = I/2;
         LaneMap[I] = I;
       }
       int j = TargetLo ? 0 : 4, je = j + 4;
       for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) {
         // Check if j is already a shuffle of this input. This happens when
         // there are two adjacent bytes after we move the low one.
         if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) {
           // If we haven't yet mapped the input, search for a slot into which
           // we can map it.
           while (j < je && PreDupI16Shuffle[j] != -1)
             ++j;
 
           if (j == je)
             // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
             return SDValue();
 
           // Map this input with the i16 shuffle.
           PreDupI16Shuffle[j] = MovingInputs[i] / 2;
         }
 
         // Update the lane map based on the mapping we ended up with.
         LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2;
       }
       V1 = DAG.getNode(
           ISD::BITCAST, DL, MVT::v16i8,
           DAG.getVectorShuffle(MVT::v8i16, DL,
                                DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1),
                                DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
 
       // Unpack the bytes to form the i16s that will be shuffled into place.
       V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
                        MVT::v16i8, V1, V1);
 
       int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
       for (int i = 0; i < 16; i += 2) {
         if (Mask[i] != -1)
           PostDupI16Shuffle[i / 2] = LaneMap[Mask[i]] - (TargetLo ? 0 : 8);
         assert(PostDupI16Shuffle[i / 2] < 8 && "Invalid v8 shuffle mask!");
       }
       return DAG.getNode(
           ISD::BITCAST, DL, MVT::v16i8,
           DAG.getVectorShuffle(MVT::v8i16, DL,
                                DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1),
                                DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
     };
     if (SDValue V = tryToWidenViaDuplication())
       return V;
   }
 
   // Check whether an interleaving lowering is likely to be more efficient.
   // This isn't perfect but it is a strong heuristic that tends to work well on
   // the kinds of shuffles that show up in practice.
   //
   // FIXME: We need to handle other interleaving widths (i16, i32, ...).
   if (shouldLowerAsInterleaving(Mask)) {
     // FIXME: Figure out whether we should pack these into the low or high
     // halves.
 
     int EMask[16], OMask[16];
     for (int i = 0; i < 8; ++i) {
       EMask[i] = Mask[2*i];
       OMask[i] = Mask[2*i + 1];
       EMask[i + 8] = -1;
       OMask[i + 8] = -1;
     }
 
     SDValue Evens = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, EMask);
     SDValue Odds = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, OMask);
 
     return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, Evens, Odds);
   }
 
   int V1LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
   int V1HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
   int V2LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
   int V2HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
 
   auto buildBlendMasks = [](MutableArrayRef<int> HalfMask,
                             MutableArrayRef<int> V1HalfBlendMask,
                             MutableArrayRef<int> V2HalfBlendMask) {
     for (int i = 0; i < 8; ++i)
       if (HalfMask[i] >= 0 && HalfMask[i] < 16) {
         V1HalfBlendMask[i] = HalfMask[i];
         HalfMask[i] = i;
       } else if (HalfMask[i] >= 16) {
         V2HalfBlendMask[i] = HalfMask[i] - 16;
         HalfMask[i] = i + 8;
       }
   };
   buildBlendMasks(LoMask, V1LoBlendMask, V2LoBlendMask);
   buildBlendMasks(HiMask, V1HiBlendMask, V2HiBlendMask);
 
   SDValue Zero = getZeroVector(MVT::v8i16, Subtarget, DAG, DL);
 
   auto buildLoAndHiV8s = [&](SDValue V, MutableArrayRef<int> LoBlendMask,
                              MutableArrayRef<int> HiBlendMask) {
     SDValue V1, V2;
     // Check if any of the odd lanes in the v16i8 are used. If not, we can mask
     // them out and avoid using UNPCK{L,H} to extract the elements of V as
     // i16s.
     if (std::none_of(LoBlendMask.begin(), LoBlendMask.end(),
                      [](int M) { return M >= 0 && M % 2 == 1; }) &&
         std::none_of(HiBlendMask.begin(), HiBlendMask.end(),
                      [](int M) { return M >= 0 && M % 2 == 1; })) {
       // Use a mask to drop the high bytes.
       V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V);
       V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, V1,
                        DAG.getConstant(0x00FF, MVT::v8i16));
 
       // This will be a single vector shuffle instead of a blend so nuke V2.
       V2 = DAG.getUNDEF(MVT::v8i16);
 
       // Squash the masks to point directly into V1.
       for (int &M : LoBlendMask)
         if (M >= 0)
           M /= 2;
       for (int &M : HiBlendMask)
         if (M >= 0)
           M /= 2;
     } else {
       // Otherwise just unpack the low half of V into V1 and the high half into
       // V2 so that we can blend them as i16s.
       V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
                        DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
       V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
                        DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
     }
 
     SDValue BlendedLo = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, LoBlendMask);
     SDValue BlendedHi = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, HiBlendMask);
     return std::make_pair(BlendedLo, BlendedHi);
   };
   SDValue V1Lo, V1Hi, V2Lo, V2Hi;
   std::tie(V1Lo, V1Hi) = buildLoAndHiV8s(V1, V1LoBlendMask, V1HiBlendMask);
   std::tie(V2Lo, V2Hi) = buildLoAndHiV8s(V2, V2LoBlendMask, V2HiBlendMask);
 
   SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, V1Lo, V2Lo, LoMask);
   SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, V1Hi, V2Hi, HiMask);
 
   return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
 }

static SDValue lowerV2F64VectorShuffle	(	SDValue	Op,
		SDValue	V1,
		SDValue	V2,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Handle lowering of 2-lane 64-bit floating point shuffles.

This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.

Definition at line 6951 of file X86ISelLowering.cpp.

                                                           {
   SDLoc DL(Op);
   assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!");
   assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   ArrayRef<int> Mask = SVOp->getMask();
   assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
 
   if (isSingleInputShuffleMask(Mask)) {
     // Straight shuffle of a single input vector. Simulate this by using the
     // single input as both of the "inputs" to this instruction..
     unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1);
     return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1,
                        DAG.getConstant(SHUFPDMask, MVT::i8));
   }
   assert(Mask[0] >= 0 && Mask[0] < 2 && "Non-canonicalized blend!");
   assert(Mask[1] >= 2 && "Non-canonicalized blend!");
 
   unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
   return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2,
                      DAG.getConstant(SHUFPDMask, MVT::i8));
 }

static SDValue lowerV2I64VectorShuffle	(	SDValue	Op,
		SDValue	V1,
		SDValue	V2,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Handle lowering of 2-lane 64-bit integer shuffles.

Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by the integer unit to minimize domain crossing penalties. However, for blends it falls back to the floating point shuffle operation with appropriate bit casting.

Definition at line 6983 of file X86ISelLowering.cpp.

                                                           {
   SDLoc DL(Op);
   assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!");
   assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   ArrayRef<int> Mask = SVOp->getMask();
   assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
 
   if (isSingleInputShuffleMask(Mask)) {
     // Straight shuffle of a single input vector. For everything from SSE2
     // onward this has a single fast instruction with no scary immediates.
     // We have to map the mask as it is actually a v4i32 shuffle instruction.
     V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1);
     int WidenedMask[4] = {
         std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1,
         std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1};
     return DAG.getNode(
         ISD::BITCAST, DL, MVT::v2i64,
         DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1,
                     getV4X86ShuffleImm8ForMask(WidenedMask, DAG)));
   }
 
   // We implement this with SHUFPD which is pretty lame because it will likely
   // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
   // However, all the alternatives are still more cycles and newer chips don't
   // have this problem. It would be really nice if x86 had better shuffles here.
   V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1);
   V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2);
   return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64,
                      DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
 }

static SDValue lowerV4F32VectorShuffle	(	SDValue	Op,
		SDValue	V1,
		SDValue	V2,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Lower 4-lane 32-bit floating point shuffles.

Uses instructions exclusively from the floating point unit to minimize domain crossing penalties, as these are sufficient to implement all v4f32 shuffles.

Definition at line 7023 of file X86ISelLowering.cpp.

                                                           {
   SDLoc DL(Op);
   assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!");
   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   ArrayRef<int> Mask = SVOp->getMask();
   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
 
   SDValue LowV = V1, HighV = V2;
   int NewMask[4] = {Mask[0], Mask[1], Mask[2], Mask[3]};
 
   int NumV2Elements =
       std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
 
   if (NumV2Elements == 0)
     // Straight shuffle of a single input vector. We pass the input vector to
     // both operands to simulate this with a SHUFPS.
     return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
                        getV4X86ShuffleImm8ForMask(Mask, DAG));
 
   if (NumV2Elements == 1) {
     int V2Index =
         std::find_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }) -
         Mask.begin();
     // Compute the index adjacent to V2Index and in the same half by toggling
     // the low bit.
     int V2AdjIndex = V2Index ^ 1;
 
     if (Mask[V2AdjIndex] == -1) {
       // Handles all the cases where we have a single V2 element and an undef.
       // This will only ever happen in the high lanes because we commute the
       // vector otherwise.
       if (V2Index < 2)
         std::swap(LowV, HighV);
       NewMask[V2Index] -= 4;
     } else {
       // Handle the case where the V2 element ends up adjacent to a V1 element.
       // To make this work, blend them together as the first step.
       int V1Index = V2AdjIndex;
       int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0};
       V2 = DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V2, V1,
                        getV4X86ShuffleImm8ForMask(BlendMask, DAG));
 
       // Now proceed to reconstruct the final blend as we have the necessary
       // high or low half formed.
       if (V2Index < 2) {
         LowV = V2;
         HighV = V1;
       } else {
         HighV = V2;
       }
       NewMask[V1Index] = 2; // We put the V1 element in V2[2].
       NewMask[V2Index] = 0; // We shifted the V2 element into V2[0].
     }
   } else if (NumV2Elements == 2) {
     if (Mask[0] < 4 && Mask[1] < 4) {
       // Handle the easy case where we have V1 in the low lanes and V2 in the
       // high lanes. We never see this reversed because we sort the shuffle.
       NewMask[2] -= 4;
       NewMask[3] -= 4;
     } else {
       // We have a mixture of V1 and V2 in both low and high lanes. Rather than
       // trying to place elements directly, just blend them and set up the final
       // shuffle to place them.
 
       // The first two blend mask elements are for V1, the second two are for
       // V2.
       int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1],
                           Mask[2] < 4 ? Mask[2] : Mask[3],
                           (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4,
                           (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4};
       V1 = DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V2,
                        getV4X86ShuffleImm8ForMask(BlendMask, DAG));
 
       // Now we do a normal shuffle of V1 by giving V1 as both operands to
       // a blend.
       LowV = HighV = V1;
       NewMask[0] = Mask[0] < 4 ? 0 : 2;
       NewMask[1] = Mask[0] < 4 ? 2 : 0;
       NewMask[2] = Mask[2] < 4 ? 1 : 3;
       NewMask[3] = Mask[2] < 4 ? 3 : 1;
     }
   }
   return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, LowV, HighV,
                      getV4X86ShuffleImm8ForMask(NewMask, DAG));
 }

static SDValue lowerV4I32VectorShuffle	(	SDValue	Op,
		SDValue	V1,
		SDValue	V2,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Lower 4-lane i32 vector shuffles.

We try to handle these with integer-domain shuffles where we can, but for blends we use the floating point domain blend instructions.

Definition at line 7117 of file X86ISelLowering.cpp.

                                                           {
   SDLoc DL(Op);
   assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!");
   assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   ArrayRef<int> Mask = SVOp->getMask();
   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
 
   if (isSingleInputShuffleMask(Mask))
     // Straight shuffle of a single input vector. For everything from SSE2
     // onward this has a single fast instruction with no scary immediates.
     return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
                        getV4X86ShuffleImm8ForMask(Mask, DAG));
 
   // We implement this with SHUFPS because it can blend from two vectors.
   // Because we're going to eventually use SHUFPS, we use SHUFPS even to build
   // up the inputs, bypassing domain shift penalties that we would encur if we
   // directly used PSHUFD on Nehalem and older. For newer chips, this isn't
   // relevant.
   return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32,
                      DAG.getVectorShuffle(
                          MVT::v4f32, DL,
                          DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1),
                          DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask));
 }

static SDValue lowerV8I16BasicBlendVectorShuffle	(	SDLoc	DL,
		SDValue	V1,
		SDValue	V2,
		MutableArrayRef< int >	Mask,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Blend two v8i16 vectors using a naive unpack strategy.

This strategy only works when the inputs from each vector fit into a single half of that vector, and generally there are not so many inputs as to leave the in-place shuffles required highly constrained (and thus expensive). It shifts all the inputs into a single side of both input vectors and then uses an unpack to interleave these inputs in a single vector. At that point, we will fall back on the generic single input shuffle lowering.

Definition at line 7474 of file X86ISelLowering.cpp.

                                                                     {
   assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
   assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
   SmallVector<int, 3> LoV1Inputs, HiV1Inputs, LoV2Inputs, HiV2Inputs;
   for (int i = 0; i < 8; ++i)
     if (Mask[i] >= 0 && Mask[i] < 4)
       LoV1Inputs.push_back(i);
     else if (Mask[i] >= 4 && Mask[i] < 8)
       HiV1Inputs.push_back(i);
     else if (Mask[i] >= 8 && Mask[i] < 12)
       LoV2Inputs.push_back(i);
     else if (Mask[i] >= 12)
       HiV2Inputs.push_back(i);
 
   int NumV1Inputs = LoV1Inputs.size() + HiV1Inputs.size();
   int NumV2Inputs = LoV2Inputs.size() + HiV2Inputs.size();
   (void)NumV1Inputs;
   (void)NumV2Inputs;
   assert(NumV1Inputs > 0 && NumV1Inputs <= 3 && "At most 3 inputs supported");
   assert(NumV2Inputs > 0 && NumV2Inputs <= 3 && "At most 3 inputs supported");
   assert(NumV1Inputs + NumV2Inputs <= 4 && "At most 4 combined inputs");
 
   bool MergeFromLo = LoV1Inputs.size() + LoV2Inputs.size() >=
                      HiV1Inputs.size() + HiV2Inputs.size();
 
   auto moveInputsToHalf = [&](SDValue V, ArrayRef<int> LoInputs,
                               ArrayRef<int> HiInputs, bool MoveToLo,
                               int MaskOffset) {
     ArrayRef<int> GoodInputs = MoveToLo ? LoInputs : HiInputs;
     ArrayRef<int> BadInputs = MoveToLo ? HiInputs : LoInputs;
     if (BadInputs.empty())
       return V;
 
     int MoveMask[] = {-1, -1, -1, -1, -1, -1, -1, -1};
     int MoveOffset = MoveToLo ? 0 : 4;
 
     if (GoodInputs.empty()) {
       for (int BadInput : BadInputs) {
         MoveMask[Mask[BadInput] % 4 + MoveOffset] = Mask[BadInput] - MaskOffset;
         Mask[BadInput] = Mask[BadInput] % 4 + MoveOffset + MaskOffset;
       }
     } else {
       if (GoodInputs.size() == 2) {
         // If the low inputs are spread across two dwords, pack them into
         // a single dword.
         MoveMask[Mask[GoodInputs[0]] % 2 + MoveOffset] =
             Mask[GoodInputs[0]] - MaskOffset;
         MoveMask[Mask[GoodInputs[1]] % 2 + MoveOffset] =
             Mask[GoodInputs[1]] - MaskOffset;
         Mask[GoodInputs[0]] = Mask[GoodInputs[0]] % 2 + MoveOffset + MaskOffset;
         Mask[GoodInputs[1]] = Mask[GoodInputs[0]] % 2 + MoveOffset + MaskOffset;
       } else {
         // Otherwise pin the low inputs.
         for (int GoodInput : GoodInputs)
           MoveMask[Mask[GoodInput] - MaskOffset] = Mask[GoodInput] - MaskOffset;
       }
 
       int MoveMaskIdx =
           std::find(std::begin(MoveMask) + MoveOffset, std::end(MoveMask), -1) -
           std::begin(MoveMask);
       assert(MoveMaskIdx >= MoveOffset && "Established above");
 
       if (BadInputs.size() == 2) {
         assert(MoveMask[MoveMaskIdx] == -1 && "Expected empty slot");
         assert(MoveMask[MoveMaskIdx + 1] == -1 && "Expected empty slot");
         MoveMask[MoveMaskIdx + Mask[BadInputs[0]] % 2] =
             Mask[BadInputs[0]] - MaskOffset;
         MoveMask[MoveMaskIdx + Mask[BadInputs[1]] % 2] =
             Mask[BadInputs[1]] - MaskOffset;
         Mask[BadInputs[0]] = MoveMaskIdx + Mask[BadInputs[0]] % 2 + MaskOffset;
         Mask[BadInputs[1]] = MoveMaskIdx + Mask[BadInputs[1]] % 2 + MaskOffset;
       } else {
         assert(BadInputs.size() == 1 && "All sizes handled");
         MoveMask[MoveMaskIdx] = Mask[BadInputs[0]] - MaskOffset;
         Mask[BadInputs[0]] = MoveMaskIdx + MaskOffset;
       }
     }
 
     return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16),
                                 MoveMask);
   };
   V1 = moveInputsToHalf(V1, LoV1Inputs, HiV1Inputs, MergeFromLo,
                         /*MaskOffset*/ 0);
   V2 = moveInputsToHalf(V2, LoV2Inputs, HiV2Inputs, MergeFromLo,
                         /*MaskOffset*/ 8);
 
   // FIXME: Select an interleaving of the merge of V1 and V2 that minimizes
   // cross-half traffic in the final shuffle.
 
   // Munge the mask to be a single-input mask after the unpack merges the
   // results.
   for (int &M : Mask)
     if (M != -1)
       M = 2 * (M % 4) + (M / 8);
 
   return DAG.getVectorShuffle(
       MVT::v8i16, DL, DAG.getNode(MergeFromLo ? X86ISD::UNPCKL : X86ISD::UNPCKH,
                                   DL, MVT::v8i16, V1, V2),
       DAG.getUNDEF(MVT::v8i16), Mask);
 }

static SDValue lowerV8I16SingleInputVectorShuffle	(	SDLoc	DL,
		SDValue	V,
		MutableArrayRef< int >	Mask,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.

The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.

The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.

Definition at line 7157 of file X86ISelLowering.cpp.

                                                       {
   assert(V.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
   MutableArrayRef<int> LoMask = Mask.slice(0, 4);
   MutableArrayRef<int> HiMask = Mask.slice(4, 4);
 
   SmallVector<int, 4> LoInputs;
   std::copy_if(LoMask.begin(), LoMask.end(), std::back_inserter(LoInputs),
                [](int M) { return M >= 0; });
   std::sort(LoInputs.begin(), LoInputs.end());
   LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), LoInputs.end());
   SmallVector<int, 4> HiInputs;
   std::copy_if(HiMask.begin(), HiMask.end(), std::back_inserter(HiInputs),
                [](int M) { return M >= 0; });
   std::sort(HiInputs.begin(), HiInputs.end());
   HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), HiInputs.end());
   int NumLToL =
       std::lower_bound(LoInputs.begin(), LoInputs.end(), 4) - LoInputs.begin();
   int NumHToL = LoInputs.size() - NumLToL;
   int NumLToH =
       std::lower_bound(HiInputs.begin(), HiInputs.end(), 4) - HiInputs.begin();
   int NumHToH = HiInputs.size() - NumLToH;
   MutableArrayRef<int> LToLInputs(LoInputs.data(), NumLToL);
   MutableArrayRef<int> LToHInputs(HiInputs.data(), NumLToH);
   MutableArrayRef<int> HToLInputs(LoInputs.data() + NumLToL, NumHToL);
   MutableArrayRef<int> HToHInputs(HiInputs.data() + NumLToH, NumHToH);
 
   // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all
   // such inputs we can swap two of the dwords across the half mark and end up
   // with <=2 inputs to each half in each half. Once there, we can fall through
   // to the generic code below. For example:
   //
   // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
   // Mask:  [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5]
   //
   // Before we had 3-1 in the low half and 3-1 in the high half. Afterward, 2-2
   // and 2-2.
   auto balanceSides = [&](ArrayRef<int> ThreeInputs, int OneInput,
                           int ThreeInputHalfSum, int OneInputHalfOffset) {
     // Compute the index of dword with only one word among the three inputs in
     // a half by taking the sum of the half with three inputs and subtracting
     // the sum of the actual three inputs. The difference is the remaining
     // slot.
     int DWordA = (ThreeInputHalfSum -
                   std::accumulate(ThreeInputs.begin(), ThreeInputs.end(), 0)) /
                  2;
     int DWordB = OneInputHalfOffset / 2 + (OneInput / 2 + 1) % 2;
 
     int PSHUFDMask[] = {0, 1, 2, 3};
     PSHUFDMask[DWordA] = DWordB;
     PSHUFDMask[DWordB] = DWordA;
     V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
                     DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
                                 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V),
                                 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
 
     // Adjust the mask to match the new locations of A and B.
     for (int &M : Mask)
       if (M != -1 && M/2 == DWordA)
         M = 2 * DWordB + M % 2;
       else if (M != -1 && M/2 == DWordB)
         M = 2 * DWordA + M % 2;
 
     // Recurse back into this routine to re-compute state now that this isn't
     // a 3 and 1 problem.
     return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16),
                                 Mask);
   };
   if (NumLToL == 3 && NumHToL == 1)
     return balanceSides(LToLInputs, HToLInputs[0], 0 + 1 + 2 + 3, 4);
   else if (NumLToL == 1 && NumHToL == 3)
     return balanceSides(HToLInputs, LToLInputs[0], 4 + 5 + 6 + 7, 0);
   else if (NumLToH == 1 && NumHToH == 3)
     return balanceSides(HToHInputs, LToHInputs[0], 4 + 5 + 6 + 7, 0);
   else if (NumLToH == 3 && NumHToH == 1)
     return balanceSides(LToHInputs, HToHInputs[0], 0 + 1 + 2 + 3, 4);
 
   // At this point there are at most two inputs to the low and high halves from
   // each half. That means the inputs can always be grouped into dwords and
   // those dwords can then be moved to the correct half with a dword shuffle.
   // We use at most one low and one high word shuffle to collect these paired
   // inputs into dwords, and finally a dword shuffle to place them.
   int PSHUFLMask[4] = {-1, -1, -1, -1};
   int PSHUFHMask[4] = {-1, -1, -1, -1};
   int PSHUFDMask[4] = {-1, -1, -1, -1};
 
   // First fix the masks for all the inputs that are staying in their
   // original halves. This will then dictate the targets of the cross-half
   // shuffles.
   auto fixInPlaceInputs = [&PSHUFDMask](
       ArrayRef<int> InPlaceInputs, MutableArrayRef<int> SourceHalfMask,
       MutableArrayRef<int> HalfMask, int HalfOffset) {
     if (InPlaceInputs.empty())
       return;
     if (InPlaceInputs.size() == 1) {
       SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
           InPlaceInputs[0] - HalfOffset;
       PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
       return;
     }
 
     assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
     SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
         InPlaceInputs[0] - HalfOffset;
     // Put the second input next to the first so that they are packed into
     // a dword. We find the adjacent index by toggling the low bit.
     int AdjIndex = InPlaceInputs[0] ^ 1;
     SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
     std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex);
     PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
   };
   if (!HToLInputs.empty())
     fixInPlaceInputs(LToLInputs, PSHUFLMask, LoMask, 0);
   if (!LToHInputs.empty())
     fixInPlaceInputs(HToHInputs, PSHUFHMask, HiMask, 4);
 
   // Now gather the cross-half inputs and place them into a free dword of
   // their target half.
   // FIXME: This operation could almost certainly be simplified dramatically to
   // look more like the 3-1 fixing operation.
   auto moveInputsToRightHalf = [&PSHUFDMask](
       MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
       MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
       int SourceOffset, int DestOffset) {
     auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
       return SourceHalfMask[Word] != -1 && SourceHalfMask[Word] != Word;
     };
     auto isDWordClobbered = [&isWordClobbered](ArrayRef<int> SourceHalfMask,
                                                int Word) {
       int LowWord = Word & ~1;
       int HighWord = Word | 1;
       return isWordClobbered(SourceHalfMask, LowWord) ||
              isWordClobbered(SourceHalfMask, HighWord);
     };
 
     if (IncomingInputs.empty())
       return;
 
     if (ExistingInputs.empty()) {
       // Map any dwords with inputs from them into the right half.
       for (int Input : IncomingInputs) {
         // If the source half mask maps over the inputs, turn those into
         // swaps and use the swapped lane.
         if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) {
           if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == -1) {
             SourceHalfMask[SourceHalfMask[Input - SourceOffset]] =
                 Input - SourceOffset;
             // We have to swap the uses in our half mask in one sweep.
             for (int &M : HalfMask)
               if (M == SourceHalfMask[Input - SourceOffset])
                 M = Input;
               else if (M == Input)
                 M = SourceHalfMask[Input - SourceOffset] + SourceOffset;
           } else {
             assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] ==
                        Input - SourceOffset &&
                    "Previous placement doesn't match!");
           }
           // Note that this correctly re-maps both when we do a swap and when
           // we observe the other side of the swap above. We rely on that to
           // avoid swapping the members of the input list directly.
           Input = SourceHalfMask[Input - SourceOffset] + SourceOffset;
         }
 
         // Map the input's dword into the correct half.
         if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == -1)
           PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2;
         else
           assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] ==
                      Input / 2 &&
                  "Previous placement doesn't match!");
       }
 
       // And just directly shift any other-half mask elements to be same-half
       // as we will have mirrored the dword containing the element into the
       // same position within that half.
       for (int &M : HalfMask)
         if (M >= SourceOffset && M < SourceOffset + 4) {
           M = M - SourceOffset + DestOffset;
           assert(M >= 0 && "This should never wrap below zero!");
         }
       return;
     }
 
     // Ensure we have the input in a viable dword of its current half. This
     // is particularly tricky because the original position may be clobbered
     // by inputs being moved and *staying* in that half.
     if (IncomingInputs.size() == 1) {
       if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
         int InputFixed = std::find(std::begin(SourceHalfMask),
                                    std::end(SourceHalfMask), -1) -
                          std::begin(SourceHalfMask) + SourceOffset;
         SourceHalfMask[InputFixed - SourceOffset] =
             IncomingInputs[0] - SourceOffset;
         std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0],
                      InputFixed);
         IncomingInputs[0] = InputFixed;
       }
     } else if (IncomingInputs.size() == 2) {
       if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 ||
           isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
         int SourceDWordBase = !isDWordClobbered(SourceHalfMask, 0) ? 0 : 2;
         assert(!isDWordClobbered(SourceHalfMask, SourceDWordBase) &&
                "Not all dwords can be clobbered!");
         SourceHalfMask[SourceDWordBase] = IncomingInputs[0] - SourceOffset;
         SourceHalfMask[SourceDWordBase + 1] = IncomingInputs[1] - SourceOffset;
         for (int &M : HalfMask)
           if (M == IncomingInputs[0])
             M = SourceDWordBase + SourceOffset;
           else if (M == IncomingInputs[1])
             M = SourceDWordBase + 1 + SourceOffset;
         IncomingInputs[0] = SourceDWordBase + SourceOffset;
         IncomingInputs[1] = SourceDWordBase + 1 + SourceOffset;
       }
     } else {
       llvm_unreachable("Unhandled input size!");
     }
 
     // Now hoist the DWord down to the right half.
     int FreeDWord = (PSHUFDMask[DestOffset / 2] == -1 ? 0 : 1) + DestOffset / 2;
     assert(PSHUFDMask[FreeDWord] == -1 && "DWord not free");
     PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2;
     for (int Input : IncomingInputs)
       std::replace(HalfMask.begin(), HalfMask.end(), Input,
                    FreeDWord * 2 + Input % 2);
   };
   moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask,
                         /*SourceOffset*/ 4, /*DestOffset*/ 0);
   moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask,
                         /*SourceOffset*/ 0, /*DestOffset*/ 4);
 
   // Now enact all the shuffles we've computed to move the inputs into their
   // target half.
   if (!isNoopShuffleMask(PSHUFLMask))
     V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V,
                     getV4X86ShuffleImm8ForMask(PSHUFLMask, DAG));
   if (!isNoopShuffleMask(PSHUFHMask))
     V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V,
                     getV4X86ShuffleImm8ForMask(PSHUFHMask, DAG));
   if (!isNoopShuffleMask(PSHUFDMask))
     V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
                     DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
                                 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V),
                                 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
 
   // At this point, each half should contain all its inputs, and we can then
   // just shuffle them into their final position.
   assert(std::count_if(LoMask.begin(), LoMask.end(),
                        [](int M) { return M >= 4; }) == 0 &&
          "Failed to lift all the high half inputs to the low mask!");
   assert(std::count_if(HiMask.begin(), HiMask.end(),
                        [](int M) { return M >= 0 && M < 4; }) == 0 &&
          "Failed to lift all the low half inputs to the high mask!");
 
   // Do a half shuffle for the low mask.
   if (!isNoopShuffleMask(LoMask))
     V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V,
                     getV4X86ShuffleImm8ForMask(LoMask, DAG));
 
   // Do a half shuffle with the high mask after shifting its values down.
   for (int &M : HiMask)
     if (M >= 0)
       M -= 4;
   if (!isNoopShuffleMask(HiMask))
     V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V,
                     getV4X86ShuffleImm8ForMask(HiMask, DAG));
 
   return V;
 }

static SDValue lowerV8I16VectorShuffle	(	SDValue	Op,
		SDValue	V1,
		SDValue	V2,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Generic lowering of 8-lane i16 shuffles.

This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.

The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.

Definition at line 7591 of file X86ISelLowering.cpp.

                                                           {
   SDLoc DL(Op);
   assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!");
   assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   ArrayRef<int> OrigMask = SVOp->getMask();
   int MaskStorage[8] = {OrigMask[0], OrigMask[1], OrigMask[2], OrigMask[3],
                         OrigMask[4], OrigMask[5], OrigMask[6], OrigMask[7]};
   MutableArrayRef<int> Mask(MaskStorage);
 
   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
 
   auto isV1 = [](int M) { return M >= 0 && M < 8; };
   auto isV2 = [](int M) { return M >= 8; };
 
   int NumV1Inputs = std::count_if(Mask.begin(), Mask.end(), isV1);
   int NumV2Inputs = std::count_if(Mask.begin(), Mask.end(), isV2);
 
   if (NumV2Inputs == 0)
     return lowerV8I16SingleInputVectorShuffle(DL, V1, Mask, Subtarget, DAG);
 
   assert(NumV1Inputs > 0 && "All single-input shuffles should be canonicalized "
                             "to be V1-input shuffles.");
 
   if (NumV1Inputs + NumV2Inputs <= 4)
     return lowerV8I16BasicBlendVectorShuffle(DL, V1, V2, Mask, Subtarget, DAG);
 
   // Check whether an interleaving lowering is likely to be more efficient.
   // This isn't perfect but it is a strong heuristic that tends to work well on
   // the kinds of shuffles that show up in practice.
   //
   // FIXME: Handle 1x, 2x, and 4x interleaving.
   if (shouldLowerAsInterleaving(Mask)) {
     // FIXME: Figure out whether we should pack these into the low or high
     // halves.
 
     int EMask[8], OMask[8];
     for (int i = 0; i < 4; ++i) {
       EMask[i] = Mask[2*i];
       OMask[i] = Mask[2*i + 1];
       EMask[i + 4] = -1;
       OMask[i + 4] = -1;
     }
 
     SDValue Evens = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, EMask);
     SDValue Odds = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, OMask);
 
     return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, Evens, Odds);
   }
 
   int LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
   int HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
 
   for (int i = 0; i < 4; ++i) {
     LoBlendMask[i] = Mask[i];
     HiBlendMask[i] = Mask[i + 4];
   }
 
   SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, LoBlendMask);
   SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, HiBlendMask);
   LoV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, LoV);
   HiV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, HiV);
 
   return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
                      DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, LoV, HiV));
 }

static SDValue LowerVACOPY	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 13356 of file X86ISelLowering.cpp.

                                               {
   // X86-64 va_list is a struct { i32, i32, i8*, i8* }.
   assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!");
   SDValue Chain = Op.getOperand(0);
   SDValue DstPtr = Op.getOperand(1);
   SDValue SrcPtr = Op.getOperand(2);
   const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
   SDLoc DL(Op);
 
   return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr,
                        DAG.getIntPtrConstant(24), 8, /*isVolatile*/false,
                        false,
                        MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
 }

static SDValue LowerVECTOR_SHUFFLE_128v4	(	ShuffleVectorSDNode *	SVOp,
		SelectionDAG &	DAG
	)

static

LowerVECTOR_SHUFFLE_128v4 - Handle all 128-bit wide vectors with 4 elements, and match them with several different shuffle types.

Definition at line 8753 of file X86ISelLowering.cpp.

                                                                         {
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   SDLoc dl(SVOp);
   MVT VT = SVOp->getSimpleValueType(0);
 
   assert(VT.is128BitVector() && "Unsupported vector size");
 
   std::pair<int, int> Locs[4];
   int Mask1[] = { -1, -1, -1, -1 };
   SmallVector<int, 8> PermMask(SVOp->getMask().begin(), SVOp->getMask().end());
 
   unsigned NumHi = 0;
   unsigned NumLo = 0;
   for (unsigned i = 0; i != 4; ++i) {
     int Idx = PermMask[i];
     if (Idx < 0) {
       Locs[i] = std::make_pair(-1, -1);
     } else {
       assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!");
       if (Idx < 4) {
         Locs[i] = std::make_pair(0, NumLo);
         Mask1[NumLo] = Idx;
         NumLo++;
       } else {
         Locs[i] = std::make_pair(1, NumHi);
         if (2+NumHi < 4)
           Mask1[2+NumHi] = Idx;
         NumHi++;
       }
     }
   }
 
   if (NumLo <= 2 && NumHi <= 2) {
     // If no more than two elements come from either vector. This can be
     // implemented with two shuffles. First shuffle gather the elements.
     // The second shuffle, which takes the first shuffle as both of its
     // vector operands, put the elements into the right order.
     V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
 
     int Mask2[] = { -1, -1, -1, -1 };
 
     for (unsigned i = 0; i != 4; ++i)
       if (Locs[i].first != -1) {
         unsigned Idx = (i < 2) ? 0 : 4;
         Idx += Locs[i].first * 2 + Locs[i].second;
         Mask2[i] = Idx;
       }
 
     return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]);
   }
 
   if (NumLo == 3 || NumHi == 3) {
     // Otherwise, we must have three elements from one vector, call it X, and
     // one element from the other, call it Y.  First, use a shufps to build an
     // intermediate vector with the one element from Y and the element from X
     // that will be in the same half in the final destination (the indexes don't
     // matter). Then, use a shufps to build the final vector, taking the half
     // containing the element from Y from the intermediate, and the other half
     // from X.
     if (NumHi == 3) {
       // Normalize it so the 3 elements come from V1.
       CommuteVectorShuffleMask(PermMask, 4);
       std::swap(V1, V2);
     }
 
     // Find the element from V2.
     unsigned HiIndex;
     for (HiIndex = 0; HiIndex < 3; ++HiIndex) {
       int Val = PermMask[HiIndex];
       if (Val < 0)
         continue;
       if (Val >= 4)
         break;
     }
 
     Mask1[0] = PermMask[HiIndex];
     Mask1[1] = -1;
     Mask1[2] = PermMask[HiIndex^1];
     Mask1[3] = -1;
     V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
 
     if (HiIndex >= 2) {
       Mask1[0] = PermMask[0];
       Mask1[1] = PermMask[1];
       Mask1[2] = HiIndex & 1 ? 6 : 4;
       Mask1[3] = HiIndex & 1 ? 4 : 6;
       return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
     }
 
     Mask1[0] = HiIndex & 1 ? 2 : 0;
     Mask1[1] = HiIndex & 1 ? 0 : 2;
     Mask1[2] = PermMask[2];
     Mask1[3] = PermMask[3];
     if (Mask1[2] >= 0)
       Mask1[2] += 4;
     if (Mask1[3] >= 0)
       Mask1[3] += 4;
     return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]);
   }
 
   // Break it into (shuffle shuffle_hi, shuffle_lo).
   int LoMask[] = { -1, -1, -1, -1 };
   int HiMask[] = { -1, -1, -1, -1 };
 
   int *MaskPtr = LoMask;
   unsigned MaskIdx = 0;
   unsigned LoIdx = 0;
   unsigned HiIdx = 2;
   for (unsigned i = 0; i != 4; ++i) {
     if (i == 2) {
       MaskPtr = HiMask;
       MaskIdx = 1;
       LoIdx = 0;
       HiIdx = 2;
     }
     int Idx = PermMask[i];
     if (Idx < 0) {
       Locs[i] = std::make_pair(-1, -1);
     } else if (Idx < 4) {
       Locs[i] = std::make_pair(MaskIdx, LoIdx);
       MaskPtr[LoIdx] = Idx;
       LoIdx++;
     } else {
       Locs[i] = std::make_pair(MaskIdx, HiIdx);
       MaskPtr[HiIdx] = Idx;
       HiIdx++;
     }
   }
 
   SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]);
   SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]);
   int MaskOps[] = { -1, -1, -1, -1 };
   for (unsigned i = 0; i != 4; ++i)
     if (Locs[i].first != -1)
       MaskOps[i] = Locs[i].first * 4 + Locs[i].second;
   return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]);
 }

static SDValue LowerVECTOR_SHUFFLE_256	(	ShuffleVectorSDNode *	SVOp,
		SelectionDAG &	DAG
	)

static

LowerVECTOR_SHUFFLE_256 - Handle all 256-bit wide vectors shuffles which could not be matched by any known target speficic shuffle

Definition at line 8640 of file X86ISelLowering.cpp.

                                                                       {
 
   SDValue NewOp = Compact8x32ShuffleNode(SVOp, DAG);
   if (NewOp.getNode())
     return NewOp;
 
   MVT VT = SVOp->getSimpleValueType(0);
 
   unsigned NumElems = VT.getVectorNumElements();
   unsigned NumLaneElems = NumElems / 2;
 
   SDLoc dl(SVOp);
   MVT EltVT = VT.getVectorElementType();
   MVT NVT = MVT::getVectorVT(EltVT, NumLaneElems);
   SDValue Output[2];
 
   SmallVector<int, 16> Mask;
   for (unsigned l = 0; l < 2; ++l) {
     // Build a shuffle mask for the output, discovering on the fly which
     // input vectors to use as shuffle operands (recorded in InputUsed).
     // If building a suitable shuffle vector proves too hard, then bail
     // out with UseBuildVector set.
     bool UseBuildVector = false;
     int InputUsed[2] = { -1, -1 }; // Not yet discovered.
     unsigned LaneStart = l * NumLaneElems;
     for (unsigned i = 0; i != NumLaneElems; ++i) {
       // The mask element.  This indexes into the input.
       int Idx = SVOp->getMaskElt(i+LaneStart);
       if (Idx < 0) {
         // the mask element does not index into any input vector.
         Mask.push_back(-1);
         continue;
       }
 
       // The input vector this mask element indexes into.
       int Input = Idx / NumLaneElems;
 
       // Turn the index into an offset from the start of the input vector.
       Idx -= Input * NumLaneElems;
 
       // Find or create a shuffle vector operand to hold this input.
       unsigned OpNo;
       for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
         if (InputUsed[OpNo] == Input)
           // This input vector is already an operand.
           break;
         if (InputUsed[OpNo] < 0) {
           // Create a new operand for this input vector.
           InputUsed[OpNo] = Input;
           break;
         }
       }
 
       if (OpNo >= array_lengthof(InputUsed)) {
         // More than two input vectors used!  Give up on trying to create a
         // shuffle vector.  Insert all elements into a BUILD_VECTOR instead.
         UseBuildVector = true;
         break;
       }
 
       // Add the mask index for the new shuffle vector.
       Mask.push_back(Idx + OpNo * NumLaneElems);
     }
 
     if (UseBuildVector) {
       SmallVector<SDValue, 16> SVOps;
       for (unsigned i = 0; i != NumLaneElems; ++i) {
         // The mask element.  This indexes into the input.
         int Idx = SVOp->getMaskElt(i+LaneStart);
         if (Idx < 0) {
           SVOps.push_back(DAG.getUNDEF(EltVT));
           continue;
         }
 
         // The input vector this mask element indexes into.
         int Input = Idx / NumElems;
 
         // Turn the index into an offset from the start of the input vector.
         Idx -= Input * NumElems;
 
         // Extract the vector element by hand.
         SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
                                     SVOp->getOperand(Input),
                                     DAG.getIntPtrConstant(Idx)));
       }
 
       // Construct the output using a BUILD_VECTOR.
       Output[l] = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, SVOps);
     } else if (InputUsed[0] < 0) {
       // No input vectors were used! The result is undefined.
       Output[l] = DAG.getUNDEF(NVT);
     } else {
       SDValue Op0 = Extract128BitVector(SVOp->getOperand(InputUsed[0] / 2),
                                         (InputUsed[0] % 2) * NumLaneElems,
                                         DAG, dl);
       // If only one input was used, use an undefined vector for the other.
       SDValue Op1 = (InputUsed[1] < 0) ? DAG.getUNDEF(NVT) :
         Extract128BitVector(SVOp->getOperand(InputUsed[1] / 2),
                             (InputUsed[1] % 2) * NumLaneElems, DAG, dl);
       // At least one input vector was used. Create a new shuffle vector.
       Output[l] = DAG.getVectorShuffle(NVT, dl, Op0, Op1, &Mask[0]);
     }
 
     Mask.clear();
   }
 
   // Concatenate the result back
   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Output[0], Output[1]);
 }

static SDValue LowerVECTOR_SHUFFLEtoBlend	(	ShuffleVectorSDNode *	SVOp,
		unsigned	MaskValue,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 8054 of file X86ISelLowering.cpp.

                                                              {
   MVT VT = SVOp->getSimpleValueType(0);
   MVT EltVT = VT.getVectorElementType();
   assert(isBlendMask(SVOp->getMask(), VT, Subtarget->hasSSE41(),
                      Subtarget->hasInt256() && "Trying to lower a "
                                                "VECTOR_SHUFFLE to a Blend but "
                                                "with the wrong mask"));
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   SDLoc dl(SVOp);
   unsigned NumElems = VT.getVectorNumElements();
 
   // Convert i32 vectors to floating point if it is not AVX2.
   // AVX2 introduced VPBLENDD instruction for 128 and 256-bit vectors.
   MVT BlendVT = VT;
   if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) {
     BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()),
                                NumElems);
     V1 = DAG.getNode(ISD::BITCAST, dl, VT, V1);
     V2 = DAG.getNode(ISD::BITCAST, dl, VT, V2);
   }
 
   SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, V1, V2,
                             DAG.getConstant(MaskValue, MVT::i32));
   return DAG.getNode(ISD::BITCAST, dl, VT, Ret);
 }

static SDValue LowerVECTOR_SHUFFLEv16i16	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

v16i16 shuffles

FIXME: We only support generation of a single pshufb currently. We can generalize the other applicable cases from LowerVECTOR_SHUFFLEv8i16 as well (e.g 2 x pshufb + 1 x por).

Definition at line 8382 of file X86ISelLowering.cpp.

                                                          {
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   SDLoc dl(SVOp);
 
   if (V2.getOpcode() != ISD::UNDEF)
     return SDValue();
 
   SmallVector<int, 16> MaskVals(SVOp->getMask().begin(), SVOp->getMask().end());
   return getPSHUFB(MaskVals, V1, dl, DAG);
 }

static SDValue LowerVECTOR_SHUFFLEv16i8	(	ShuffleVectorSDNode *	SVOp,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 8399 of file X86ISelLowering.cpp.

                                                            {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   SDLoc dl(SVOp);
   ArrayRef<int> MaskVals = SVOp->getMask();
 
   // Promote splats to a larger type which usually leads to more efficient code.
   // FIXME: Is this true if pshufb is available?
   if (SVOp->isSplat())
     return PromoteSplat(SVOp, DAG);
 
   // If we have SSSE3, case 1 is generated when all result bytes come from
   // one of  the inputs.  Otherwise, case 2 is generated.  If no SSSE3 is
   // present, fall back to case 3.
 
   // If SSSE3, use 1 pshufb instruction per vector with elements in the result.
   if (Subtarget->hasSSSE3()) {
     SmallVector<SDValue,16> pshufbMask;
 
     // If all result elements are from one input vector, then only translate
     // undef mask values to 0x80 (zero out result) in the pshufb mask.
     //
     // Otherwise, we have elements from both input vectors, and must zero out
     // elements that come from V2 in the first mask, and V1 in the second mask
     // so that we can OR them together.
     for (unsigned i = 0; i != 16; ++i) {
       int EltIdx = MaskVals[i];
       if (EltIdx < 0 || EltIdx >= 16)
         EltIdx = 0x80;
       pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
     }
     V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
                      DAG.getNode(ISD::BUILD_VECTOR, dl,
                                  MVT::v16i8, pshufbMask));
 
     // As PSHUFB will zero elements with negative indices, it's safe to ignore
     // the 2nd operand if it's undefined or zero.
     if (V2.getOpcode() == ISD::UNDEF ||
         ISD::isBuildVectorAllZeros(V2.getNode()))
       return V1;
 
     // Calculate the shuffle mask for the second input, shuffle it, and
     // OR it with the first shuffled input.
     pshufbMask.clear();
     for (unsigned i = 0; i != 16; ++i) {
       int EltIdx = MaskVals[i];
       EltIdx = (EltIdx < 16) ? 0x80 : EltIdx - 16;
       pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
     }
     V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
                      DAG.getNode(ISD::BUILD_VECTOR, dl,
                                  MVT::v16i8, pshufbMask));
     return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
   }
 
   // No SSSE3 - Calculate in place words and then fix all out of place words
   // With 0-16 extracts & inserts.  Worst case is 16 bytes out of order from
   // the 16 different words that comprise the two doublequadword input vectors.
   V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
   V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2);
   SDValue NewV = V1;
   for (int i = 0; i != 8; ++i) {
     int Elt0 = MaskVals[i*2];
     int Elt1 = MaskVals[i*2+1];
 
     // This word of the result is all undef, skip it.
     if (Elt0 < 0 && Elt1 < 0)
       continue;
 
     // This word of the result is already in the correct place, skip it.
     if ((Elt0 == i*2) && (Elt1 == i*2+1))
       continue;
 
     SDValue Elt0Src = Elt0 < 16 ? V1 : V2;
     SDValue Elt1Src = Elt1 < 16 ? V1 : V2;
     SDValue InsElt;
 
     // If Elt0 and Elt1 are defined, are consecutive, and can be load
     // using a single extract together, load it and store it.
     if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) {
       InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
                            DAG.getIntPtrConstant(Elt1 / 2));
       NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
                         DAG.getIntPtrConstant(i));
       continue;
     }
 
     // If Elt1 is defined, extract it from the appropriate source.  If the
     // source byte is not also odd, shift the extracted word left 8 bits
     // otherwise clear the bottom 8 bits if we need to do an or.
     if (Elt1 >= 0) {
       InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
                            DAG.getIntPtrConstant(Elt1 / 2));
       if ((Elt1 & 1) == 0)
         InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt,
                              DAG.getConstant(8,
                                   TLI.getShiftAmountTy(InsElt.getValueType())));
       else if (Elt0 >= 0)
         InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt,
                              DAG.getConstant(0xFF00, MVT::i16));
     }
     // If Elt0 is defined, extract it from the appropriate source.  If the
     // source byte is not also even, shift the extracted word right 8 bits. If
     // Elt1 was also defined, OR the extracted values together before
     // inserting them in the result.
     if (Elt0 >= 0) {
       SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
                                     Elt0Src, DAG.getIntPtrConstant(Elt0 / 2));
       if ((Elt0 & 1) != 0)
         InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0,
                               DAG.getConstant(8,
                                  TLI.getShiftAmountTy(InsElt0.getValueType())));
       else if (Elt1 >= 0)
         InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0,
                              DAG.getConstant(0x00FF, MVT::i16));
       InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0)
                          : InsElt0;
     }
     NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
                        DAG.getIntPtrConstant(i));
   }
   return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV);
 }

static SDValue LowerVECTOR_SHUFFLEv32i8	(	ShuffleVectorSDNode *	SVOp,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 8528 of file X86ISelLowering.cpp.

                                                     {
   MVT VT = SVOp->getSimpleValueType(0);
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   SDLoc dl(SVOp);
   SmallVector<int, 32> MaskVals(SVOp->getMask().begin(), SVOp->getMask().end());
 
   bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
   bool V1IsAllZero = ISD::isBuildVectorAllZeros(V1.getNode());
   bool V2IsAllZero = ISD::isBuildVectorAllZeros(V2.getNode());
 
   // VPSHUFB may be generated if
   // (1) one of input vector is undefined or zeroinitializer.
   // The mask value 0x80 puts 0 in the corresponding slot of the vector.
   // And (2) the mask indexes don't cross the 128-bit lane.
   if (VT != MVT::v32i8 || !Subtarget->hasInt256() ||
       (!V2IsUndef && !V2IsAllZero && !V1IsAllZero))
     return SDValue();
 
   if (V1IsAllZero && !V2IsAllZero) {
     CommuteVectorShuffleMask(MaskVals, 32);
     V1 = V2;
   }
   return getPSHUFB(MaskVals, V1, dl, DAG);
 }

static SDValue LowerVECTOR_SHUFFLEv8i16	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 8142 of file X86ISelLowering.cpp.

                                             {
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   SDLoc dl(SVOp);
   SmallVector<int, 8> MaskVals;
 
   // Determine if more than 1 of the words in each of the low and high quadwords
   // of the result come from the same quadword of one of the two inputs.  Undef
   // mask values count as coming from any quadword, for better codegen.
   //
   // Lo/HiQuad[i] = j indicates how many words from the ith quad of the input
   // feeds this quad.  For i, 0 and 1 refer to V1, 2 and 3 refer to V2.
   unsigned LoQuad[] = { 0, 0, 0, 0 };
   unsigned HiQuad[] = { 0, 0, 0, 0 };
   // Indices of quads used.
   std::bitset<4> InputQuads;
   for (unsigned i = 0; i < 8; ++i) {
     unsigned *Quad = i < 4 ? LoQuad : HiQuad;
     int EltIdx = SVOp->getMaskElt(i);
     MaskVals.push_back(EltIdx);
     if (EltIdx < 0) {
       ++Quad[0];
       ++Quad[1];
       ++Quad[2];
       ++Quad[3];
       continue;
     }
     ++Quad[EltIdx / 4];
     InputQuads.set(EltIdx / 4);
   }
 
   int BestLoQuad = -1;
   unsigned MaxQuad = 1;
   for (unsigned i = 0; i < 4; ++i) {
     if (LoQuad[i] > MaxQuad) {
       BestLoQuad = i;
       MaxQuad = LoQuad[i];
     }
   }
 
   int BestHiQuad = -1;
   MaxQuad = 1;
   for (unsigned i = 0; i < 4; ++i) {
     if (HiQuad[i] > MaxQuad) {
       BestHiQuad = i;
       MaxQuad = HiQuad[i];
     }
   }
 
   // For SSSE3, If all 8 words of the result come from only 1 quadword of each
   // of the two input vectors, shuffle them into one input vector so only a
   // single pshufb instruction is necessary. If there are more than 2 input
   // quads, disable the next transformation since it does not help SSSE3.
   bool V1Used = InputQuads[0] || InputQuads[1];
   bool V2Used = InputQuads[2] || InputQuads[3];
   if (Subtarget->hasSSSE3()) {
     if (InputQuads.count() == 2 && V1Used && V2Used) {
       BestLoQuad = InputQuads[0] ? 0 : 1;
       BestHiQuad = InputQuads[2] ? 2 : 3;
     }
     if (InputQuads.count() > 2) {
       BestLoQuad = -1;
       BestHiQuad = -1;
     }
   }
 
   // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update
   // the shuffle mask.  If a quad is scored as -1, that means that it contains
   // words from all 4 input quadwords.
   SDValue NewV;
   if (BestLoQuad >= 0 || BestHiQuad >= 0) {
     int MaskV[] = {
       BestLoQuad < 0 ? 0 : BestLoQuad,
       BestHiQuad < 0 ? 1 : BestHiQuad
     };
     NewV = DAG.getVectorShuffle(MVT::v2i64, dl,
                   DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1),
                   DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2), &MaskV[0]);
     NewV = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, NewV);
 
     // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the
     // source words for the shuffle, to aid later transformations.
     bool AllWordsInNewV = true;
     bool InOrder[2] = { true, true };
     for (unsigned i = 0; i != 8; ++i) {
       int idx = MaskVals[i];
       if (idx != (int)i)
         InOrder[i/4] = false;
       if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad)
         continue;
       AllWordsInNewV = false;
       break;
     }
 
     bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV;
     if (AllWordsInNewV) {
       for (int i = 0; i != 8; ++i) {
         int idx = MaskVals[i];
         if (idx < 0)
           continue;
         idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4;
         if ((idx != i) && idx < 4)
           pshufhw = false;
         if ((idx != i) && idx > 3)
           pshuflw = false;
       }
       V1 = NewV;
       V2Used = false;
       BestLoQuad = 0;
       BestHiQuad = 1;
     }
 
     // If we've eliminated the use of V2, and the new mask is a pshuflw or
     // pshufhw, that's as cheap as it gets.  Return the new shuffle.
     if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) {
       unsigned Opc = pshufhw ? X86ISD::PSHUFHW : X86ISD::PSHUFLW;
       unsigned TargetMask = 0;
       NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV,
                                   DAG.getUNDEF(MVT::v8i16), &MaskVals[0]);
       ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
       TargetMask = pshufhw ? getShufflePSHUFHWImmediate(SVOp):
                              getShufflePSHUFLWImmediate(SVOp);
       V1 = NewV.getOperand(0);
       return getTargetShuffleNode(Opc, dl, MVT::v8i16, V1, TargetMask, DAG);
     }
   }
 
   // Promote splats to a larger type which usually leads to more efficient code.
   // FIXME: Is this true if pshufb is available?
   if (SVOp->isSplat())
     return PromoteSplat(SVOp, DAG);
 
   // If we have SSSE3, and all words of the result are from 1 input vector,
   // case 2 is generated, otherwise case 3 is generated.  If no SSSE3
   // is present, fall back to case 4.
   if (Subtarget->hasSSSE3()) {
     SmallVector<SDValue,16> pshufbMask;
 
     // If we have elements from both input vectors, set the high bit of the
     // shuffle mask element to zero out elements that come from V2 in the V1
     // mask, and elements that come from V1 in the V2 mask, so that the two
     // results can be OR'd together.
     bool TwoInputs = V1Used && V2Used;
     V1 = getPSHUFB(MaskVals, V1, dl, DAG);
     if (!TwoInputs)
       return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
 
     // Calculate the shuffle mask for the second input, shuffle it, and
     // OR it with the first shuffled input.
     CommuteVectorShuffleMask(MaskVals, 8);
     V2 = getPSHUFB(MaskVals, V2, dl, DAG);
     V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
     return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
   }
 
   // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order,
   // and update MaskVals with new element order.
   std::bitset<8> InOrder;
   if (BestLoQuad >= 0) {
     int MaskV[] = { -1, -1, -1, -1, 4, 5, 6, 7 };
     for (int i = 0; i != 4; ++i) {
       int idx = MaskVals[i];
       if (idx < 0) {
         InOrder.set(i);
       } else if ((idx / 4) == BestLoQuad) {
         MaskV[i] = idx & 3;
         InOrder.set(i);
       }
     }
     NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
                                 &MaskV[0]);
 
     if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSE2()) {
       ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
       NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16,
                                   NewV.getOperand(0),
                                   getShufflePSHUFLWImmediate(SVOp), DAG);
     }
   }
 
   // If BestHi >= 0, generate a pshufhw to put the high elements in order,
   // and update MaskVals with the new element order.
   if (BestHiQuad >= 0) {
     int MaskV[] = { 0, 1, 2, 3, -1, -1, -1, -1 };
     for (unsigned i = 4; i != 8; ++i) {
       int idx = MaskVals[i];
       if (idx < 0) {
         InOrder.set(i);
       } else if ((idx / 4) == BestHiQuad) {
         MaskV[i] = (idx & 3) + 4;
         InOrder.set(i);
       }
     }
     NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
                                 &MaskV[0]);
 
     if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSE2()) {
       ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
       NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16,
                                   NewV.getOperand(0),
                                   getShufflePSHUFHWImmediate(SVOp), DAG);
     }
   }
 
   // In case BestHi & BestLo were both -1, which means each quadword has a word
   // from each of the four input quadwords, calculate the InOrder bitvector now
   // before falling through to the insert/extract cleanup.
   if (BestLoQuad == -1 && BestHiQuad == -1) {
     NewV = V1;
     for (int i = 0; i != 8; ++i)
       if (MaskVals[i] < 0 || MaskVals[i] == i)
         InOrder.set(i);
   }
 
   // The other elements are put in the right place using pextrw and pinsrw.
   for (unsigned i = 0; i != 8; ++i) {
     if (InOrder[i])
       continue;
     int EltIdx = MaskVals[i];
     if (EltIdx < 0)
       continue;
     SDValue ExtOp = (EltIdx < 8) ?
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1,
                   DAG.getIntPtrConstant(EltIdx)) :
       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2,
                   DAG.getIntPtrConstant(EltIdx - 8));
     NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp,
                        DAG.getIntPtrConstant(i));
   }
   return NewV;
 }

static SDValue LowerVectorAllZeroTest	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 11542 of file X86ISelLowering.cpp.

                                                          {
   assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree.");
 
   if (!Subtarget->hasSSE41())
     return SDValue();
 
   if (!Op->hasOneUse())
     return SDValue();
 
   SDNode *N = Op.getNode();
   SDLoc DL(N);
 
   SmallVector<SDValue, 8> Opnds;
   DenseMap<SDValue, unsigned> VecInMap;
   SmallVector<SDValue, 8> VecIns;
   EVT VT = MVT::Other;
 
   // Recognize a special case where a vector is casted into wide integer to
   // test all 0s.
   Opnds.push_back(N->getOperand(0));
   Opnds.push_back(N->getOperand(1));
 
   for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) {
     SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
     // BFS traverse all OR'd operands.
     if (I->getOpcode() == ISD::OR) {
       Opnds.push_back(I->getOperand(0));
       Opnds.push_back(I->getOperand(1));
       // Re-evaluate the number of nodes to be traversed.
       e += 2; // 2 more nodes (LHS and RHS) are pushed.
       continue;
     }
 
     // Quit if a non-EXTRACT_VECTOR_ELT
     if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
       return SDValue();
 
     // Quit if without a constant index.
     SDValue Idx = I->getOperand(1);
     if (!isa<ConstantSDNode>(Idx))
       return SDValue();
 
     SDValue ExtractedFromVec = I->getOperand(0);
     DenseMap<SDValue, unsigned>::iterator M = VecInMap.find(ExtractedFromVec);
     if (M == VecInMap.end()) {
       VT = ExtractedFromVec.getValueType();
       // Quit if not 128/256-bit vector.
       if (!VT.is128BitVector() && !VT.is256BitVector())
         return SDValue();
       // Quit if not the same type.
       if (VecInMap.begin() != VecInMap.end() &&
           VT != VecInMap.begin()->first.getValueType())
         return SDValue();
       M = VecInMap.insert(std::make_pair(ExtractedFromVec, 0)).first;
       VecIns.push_back(ExtractedFromVec);
     }
     M->second |= 1U << cast<ConstantSDNode>(Idx)->getZExtValue();
   }
 
   assert((VT.is128BitVector() || VT.is256BitVector()) &&
          "Not extracted from 128-/256-bit vector.");
 
   unsigned FullMask = (1U << VT.getVectorNumElements()) - 1U;
 
   for (DenseMap<SDValue, unsigned>::const_iterator
         I = VecInMap.begin(), E = VecInMap.end(); I != E; ++I) {
     // Quit if not all elements are used.
     if (I->second != FullMask)
       return SDValue();
   }
 
   EVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
 
   // Cast all vectors into TestVT for PTEST.
   for (unsigned i = 0, e = VecIns.size(); i < e; ++i)
     VecIns[i] = DAG.getNode(ISD::BITCAST, DL, TestVT, VecIns[i]);
 
   // If more than one full vectors are evaluated, OR them first before PTEST.
   for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1; Slot += 2, e += 1) {
     // Each iteration will OR 2 nodes and append the result until there is only
     // 1 node left, i.e. the final OR'd value of all vectors.
     SDValue LHS = VecIns[Slot];
     SDValue RHS = VecIns[Slot + 1];
     VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS));
   }
 
   return DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
                      VecIns.back(), VecIns.back());
 }

static SDValue LowerVectorBroadcast	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

LowerVectorBroadcast - Attempt to use the vbroadcast instruction to generate a splat value for the following cases:

A splat BUILD_VECTOR which uses a single scalar load, or a constant.
A splat shuffle which uses a scalar_to_vector node which comes from a scalar load, or a constant. The VBROADCAST node is returned when a pattern is found, or SDValue() otherwise.

Definition at line 5743 of file X86ISelLowering.cpp.

                                                        {
   if (!Subtarget->hasFp256())
     return SDValue();
 
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
 
   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
          "Unsupported vector type for broadcast.");
 
   SDValue Ld;
   bool ConstSplatVal;
 
   switch (Op.getOpcode()) {
     default:
       // Unknown pattern found.
       return SDValue();
 
     case ISD::BUILD_VECTOR: {
       auto *BVOp = cast<BuildVectorSDNode>(Op.getNode());
       BitVector UndefElements;
       SDValue Splat = BVOp->getSplatValue(&UndefElements);
 
       // We need a splat of a single value to use broadcast, and it doesn't
       // make any sense if the value is only in one element of the vector.
       if (!Splat || (VT.getVectorNumElements() - UndefElements.count()) <= 1)
         return SDValue();
 
       Ld = Splat;
       ConstSplatVal = (Ld.getOpcode() == ISD::Constant ||
                        Ld.getOpcode() == ISD::ConstantFP);
 
       // Make sure that all of the users of a non-constant load are from the
       // BUILD_VECTOR node.
       if (!ConstSplatVal && !BVOp->isOnlyUserOf(Ld.getNode()))
         return SDValue();
       break;
     }
 
     case ISD::VECTOR_SHUFFLE: {
       ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
 
       // Shuffles must have a splat mask where the first element is
       // broadcasted.
       if ((!SVOp->isSplat()) || SVOp->getMaskElt(0) != 0)
         return SDValue();
 
       SDValue Sc = Op.getOperand(0);
       if (Sc.getOpcode() != ISD::SCALAR_TO_VECTOR &&
           Sc.getOpcode() != ISD::BUILD_VECTOR) {
 
         if (!Subtarget->hasInt256())
           return SDValue();
 
         // Use the register form of the broadcast instruction available on AVX2.
         if (VT.getSizeInBits() >= 256)
           Sc = Extract128BitVector(Sc, 0, DAG, dl);
         return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Sc);
       }
 
       Ld = Sc.getOperand(0);
       ConstSplatVal = (Ld.getOpcode() == ISD::Constant ||
                        Ld.getOpcode() == ISD::ConstantFP);
 
       // The scalar_to_vector node and the suspected
       // load node must have exactly one user.
       // Constants may have multiple users.
 
       // AVX-512 has register version of the broadcast
       bool hasRegVer = Subtarget->hasAVX512() && VT.is512BitVector() &&
         Ld.getValueType().getSizeInBits() >= 32;
       if (!ConstSplatVal && ((!Sc.hasOneUse() || !Ld.hasOneUse()) &&
           !hasRegVer))
         return SDValue();
       break;
     }
   }
 
   bool IsGE256 = (VT.getSizeInBits() >= 256);
 
   // Handle the broadcasting a single constant scalar from the constant pool
   // into a vector. On Sandybridge it is still better to load a constant vector
   // from the constant pool and not to broadcast it from a scalar.
   if (ConstSplatVal && Subtarget->hasInt256()) {
     EVT CVT = Ld.getValueType();
     assert(!CVT.isVector() && "Must not broadcast a vector type");
     unsigned ScalarSize = CVT.getSizeInBits();
 
     if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64)) {
       const Constant *C = nullptr;
       if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Ld))
         C = CI->getConstantIntValue();
       else if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Ld))
         C = CF->getConstantFPValue();
 
       assert(C && "Invalid constant type");
 
       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
       SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
       unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
       Ld = DAG.getLoad(CVT, dl, DAG.getEntryNode(), CP,
                        MachinePointerInfo::getConstantPool(),
                        false, false, false, Alignment);
 
       return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
     }
   }
 
   bool IsLoad = ISD::isNormalLoad(Ld.getNode());
   unsigned ScalarSize = Ld.getValueType().getSizeInBits();
 
   // Handle AVX2 in-register broadcasts.
   if (!IsLoad && Subtarget->hasInt256() &&
       (ScalarSize == 32 || (IsGE256 && ScalarSize == 64)))
     return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
 
   // The scalar source must be a normal load.
   if (!IsLoad)
     return SDValue();
 
   if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64))
     return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
 
   // The integer check is needed for the 64-bit into 128-bit so it doesn't match
   // double since there is no vbroadcastsd xmm
   if (Subtarget->hasInt256() && Ld.getValueType().isInteger()) {
     if (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64)
       return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
   }
 
   // Unsupported broadcast.
   return SDValue();
 }

static SDValue LowerVectorIntExtend	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 9114 of file X86ISelLowering.cpp.

                                                        {
   // PMOVZX is only available from SSE41.
   if (!Subtarget->hasSSE41())
     return SDValue();
 
   MVT VT = Op.getSimpleValueType();
 
   // Only AVX2 support 256-bit vector integer extending.
   if (!Subtarget->hasInt256() && VT.is256BitVector())
     return SDValue();
 
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   SDLoc DL(Op);
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   unsigned NumElems = VT.getVectorNumElements();
 
   // Extending is an unary operation and the element type of the source vector
   // won't be equal to or larger than i64.
   if (V2.getOpcode() != ISD::UNDEF || !VT.isInteger() ||
       VT.getVectorElementType() == MVT::i64)
     return SDValue();
 
   // Find the expansion ratio, e.g. expanding from i8 to i32 has a ratio of 4.
   unsigned Shift = 1; // Start from 2, i.e. 1 << 1.
   while ((1U << Shift) < NumElems) {
     if (SVOp->getMaskElt(1U << Shift) == 1)
       break;
     Shift += 1;
     // The maximal ratio is 8, i.e. from i8 to i64.
     if (Shift > 3)
       return SDValue();
   }
 
   // Check the shuffle mask.
   unsigned Mask = (1U << Shift) - 1;
   for (unsigned i = 0; i != NumElems; ++i) {
     int EltIdx = SVOp->getMaskElt(i);
     if ((i & Mask) != 0 && EltIdx != -1)
       return SDValue();
     if ((i & Mask) == 0 && (unsigned)EltIdx != (i >> Shift))
       return SDValue();
   }
 
   unsigned NBits = VT.getVectorElementType().getSizeInBits() << Shift;
   MVT NeVT = MVT::getIntegerVT(NBits);
   MVT NVT = MVT::getVectorVT(NeVT, NumElems >> Shift);
 
   if (!DAG.getTargetLoweringInfo().isTypeLegal(NVT))
     return SDValue();
 
   // Simplify the operand as it's prepared to be fed into shuffle.
   unsigned SignificantBits = NVT.getSizeInBits() >> Shift;
   if (V1.getOpcode() == ISD::BITCAST &&
       V1.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR &&
       V1.getOperand(0).getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
       V1.getOperand(0).getOperand(0)
         .getSimpleValueType().getSizeInBits() == SignificantBits) {
     // (bitcast (sclr2vec (ext_vec_elt x))) -> (bitcast x)
     SDValue V = V1.getOperand(0).getOperand(0).getOperand(0);
     ConstantSDNode *CIdx =
       dyn_cast<ConstantSDNode>(V1.getOperand(0).getOperand(0).getOperand(1));
     // If it's foldable, i.e. normal load with single use, we will let code
     // selection to fold it. Otherwise, we will short the conversion sequence.
     if (CIdx && CIdx->getZExtValue() == 0 &&
         (!ISD::isNormalLoad(V.getNode()) || !V.hasOneUse())) {
       MVT FullVT = V.getSimpleValueType();
       MVT V1VT = V1.getSimpleValueType();
       if (FullVT.getSizeInBits() > V1VT.getSizeInBits()) {
         // The "ext_vec_elt" node is wider than the result node.
         // In this case we should extract subvector from V.
         // (bitcast (sclr2vec (ext_vec_elt x))) -> (bitcast (extract_subvector x)).
         unsigned Ratio = FullVT.getSizeInBits() / V1VT.getSizeInBits();
         MVT SubVecVT = MVT::getVectorVT(FullVT.getVectorElementType(),
                                         FullVT.getVectorNumElements()/Ratio);
         V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, V,
                         DAG.getIntPtrConstant(0));
       }
       V1 = DAG.getNode(ISD::BITCAST, DL, V1VT, V);
     }
   }
 
   return DAG.getNode(ISD::BITCAST, DL, VT,
                      DAG.getNode(X86ISD::VZEXT, DL, NVT, V1));
 }

static SDValue lowerVectorShuffle	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Top-level lowering for x86 vector shuffles.

This handles decomposition, canonicalization, and lowering of all x86 vector shuffles. Most of the specific lowering strategies are encapsulated above in helper routines. The canonicalization attempts to widen shuffles to involve fewer lanes of wider elements, consolidate symmetric patterns s.t. only one of the two inputs needs to be tested, etc.

Definition at line 7912 of file X86ISelLowering.cpp.

                                                      {
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   ArrayRef<int> Mask = SVOp->getMask();
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
   MVT VT = Op.getSimpleValueType();
   int NumElements = VT.getVectorNumElements();
   SDLoc dl(Op);
 
   assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles");
 
   bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
   bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
   if (V1IsUndef && V2IsUndef)
     return DAG.getUNDEF(VT);
 
   // When we create a shuffle node we put the UNDEF node to second operand,
   // but in some cases the first operand may be transformed to UNDEF.
   // In this case we should just commute the node.
   if (V1IsUndef)
     return DAG.getCommutedVectorShuffle(*SVOp);
 
   // Check for non-undef masks pointing at an undef vector and make the masks
   // undef as well. This makes it easier to match the shuffle based solely on
   // the mask.
   if (V2IsUndef)
     for (int M : Mask)
       if (M >= NumElements) {
         SmallVector<int, 8> NewMask(Mask.begin(), Mask.end());
         for (int &M : NewMask)
           if (M >= NumElements)
             M = -1;
         return DAG.getVectorShuffle(VT, dl, V1, V2, NewMask);
       }
 
   // For integer vector shuffles, try to collapse them into a shuffle of fewer
   // lanes but wider integers. We cap this to not form integers larger than i64
   // but it might be interesting to form i128 integers to handle flipping the
   // low and high halves of AVX 256-bit vectors.
   if (VT.isInteger() && VT.getScalarSizeInBits() < 64 &&
       areAdjacentMasksSequential(Mask)) {
     SmallVector<int, 8> NewMask;
     for (int i = 0, Size = Mask.size(); i < Size; i += 2)
       NewMask.push_back(Mask[i] / 2);
     MVT NewVT =
         MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits() * 2),
                          VT.getVectorNumElements() / 2);
     V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1);
     V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2);
     return DAG.getNode(ISD::BITCAST, dl, VT,
                        DAG.getVectorShuffle(NewVT, dl, V1, V2, NewMask));
   }
 
   int NumV1Elements = 0, NumUndefElements = 0, NumV2Elements = 0;
   for (int M : SVOp->getMask())
     if (M < 0)
       ++NumUndefElements;
     else if (M < NumElements)
       ++NumV1Elements;
     else
       ++NumV2Elements;
 
   // Commute the shuffle as needed such that more elements come from V1 than
   // V2. This allows us to match the shuffle pattern strictly on how many
   // elements come from V1 without handling the symmetric cases.
   if (NumV2Elements > NumV1Elements)
     return DAG.getCommutedVectorShuffle(*SVOp);
 
   // When the number of V1 and V2 elements are the same, try to minimize the
   // number of uses of V2 in the low half of the vector.
   if (NumV1Elements == NumV2Elements) {
     int LowV1Elements = 0, LowV2Elements = 0;
     for (int M : SVOp->getMask().slice(0, NumElements / 2))
       if (M >= NumElements)
         ++LowV2Elements;
       else if (M >= 0)
         ++LowV1Elements;
     if (LowV2Elements > LowV1Elements)
       return DAG.getCommutedVectorShuffle(*SVOp);
   }
 
   // For each vector width, delegate to a specialized lowering routine.
   if (VT.getSizeInBits() == 128)
     return lower128BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG);
 
   llvm_unreachable("Unimplemented!");
 }

static SDValue LowerVSELECTtoBlend	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 9643 of file X86ISelLowering.cpp.

                                                       {
   SDValue Cond = Op.getOperand(0);
   SDValue LHS = Op.getOperand(1);
   SDValue RHS = Op.getOperand(2);
   SDLoc dl(Op);
   MVT VT = Op.getSimpleValueType();
   MVT EltVT = VT.getVectorElementType();
   unsigned NumElems = VT.getVectorNumElements();
 
   // There is no blend with immediate in AVX-512.
   if (VT.is512BitVector())
     return SDValue();
 
   if (!Subtarget->hasSSE41() || EltVT == MVT::i8)
     return SDValue();
   if (!Subtarget->hasInt256() && VT == MVT::v16i16)
     return SDValue();
 
   if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
     return SDValue();
 
   // Check the mask for BLEND and build the value.
   unsigned MaskValue = 0;
   if (!BUILD_VECTORtoBlendMask(cast<BuildVectorSDNode>(Cond), MaskValue))
     return SDValue();
 
   // Convert i32 vectors to floating point if it is not AVX2.
   // AVX2 introduced VPBLENDD instruction for 128 and 256-bit vectors.
   MVT BlendVT = VT;
   if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) {
     BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()),
                                NumElems);
     LHS = DAG.getNode(ISD::BITCAST, dl, VT, LHS);
     RHS = DAG.getNode(ISD::BITCAST, dl, VT, RHS);
   }
 
   SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, LHS, RHS,
                             DAG.getConstant(MaskValue, MVT::i32));
   return DAG.getNode(ISD::BITCAST, dl, VT, Ret);
 }

static SDValue LowerVSETCC	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 12170 of file X86ISelLowering.cpp.

                                               {
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   SDValue CC = Op.getOperand(2);
   MVT VT = Op.getSimpleValueType();
   ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
   bool isFP = Op.getOperand(1).getSimpleValueType().isFloatingPoint();
   SDLoc dl(Op);
 
   if (isFP) {
 #ifndef NDEBUG
     MVT EltVT = Op0.getSimpleValueType().getVectorElementType();
     assert(EltVT == MVT::f32 || EltVT == MVT::f64);
 #endif
 
     unsigned SSECC = translateX86FSETCC(SetCCOpcode, Op0, Op1);
     unsigned Opc = X86ISD::CMPP;
     if (Subtarget->hasAVX512() && VT.getVectorElementType() == MVT::i1) {
       assert(VT.getVectorNumElements() <= 16);
       Opc = X86ISD::CMPM;
     }
     // In the two special cases we can't handle, emit two comparisons.
     if (SSECC == 8) {
       unsigned CC0, CC1;
       unsigned CombineOpc;
       if (SetCCOpcode == ISD::SETUEQ) {
         CC0 = 3; CC1 = 0; CombineOpc = ISD::OR;
       } else {
         assert(SetCCOpcode == ISD::SETONE);
         CC0 = 7; CC1 = 4; CombineOpc = ISD::AND;
       }
 
       SDValue Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,
                                  DAG.getConstant(CC0, MVT::i8));
       SDValue Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,
                                  DAG.getConstant(CC1, MVT::i8));
       return DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
     }
     // Handle all other FP comparisons here.
     return DAG.getNode(Opc, dl, VT, Op0, Op1,
                        DAG.getConstant(SSECC, MVT::i8));
   }
 
   // Break 256-bit integer vector compare into smaller ones.
   if (VT.is256BitVector() && !Subtarget->hasInt256())
     return Lower256IntVSETCC(Op, DAG);
 
   bool MaskResult = (VT.getVectorElementType() == MVT::i1);
   EVT OpVT = Op1.getValueType();
   if (Subtarget->hasAVX512()) {
     if (Op1.getValueType().is512BitVector() ||
         (MaskResult && OpVT.getVectorElementType().getSizeInBits() >= 32))
       return LowerIntVSETCC_AVX512(Op, DAG, Subtarget);
 
     // In AVX-512 architecture setcc returns mask with i1 elements,
     // But there is no compare instruction for i8 and i16 elements.
     // We are not talking about 512-bit operands in this case, these
     // types are illegal.
     if (MaskResult &&
         (OpVT.getVectorElementType().getSizeInBits() < 32 &&
          OpVT.getVectorElementType().getSizeInBits() >= 8))
       return DAG.getNode(ISD::TRUNCATE, dl, VT,
                          DAG.getNode(ISD::SETCC, dl, OpVT, Op0, Op1, CC));
   }
 
   // We are handling one of the integer comparisons here.  Since SSE only has
   // GT and EQ comparisons for integer, swapping operands and multiple
   // operations may be required for some comparisons.
   unsigned Opc;
   bool Swap = false, Invert = false, FlipSigns = false, MinMax = false;
   bool Subus = false;
 
   switch (SetCCOpcode) {
   default: llvm_unreachable("Unexpected SETCC condition");
   case ISD::SETNE:  Invert = true;
   case ISD::SETEQ:  Opc = X86ISD::PCMPEQ; break;
   case ISD::SETLT:  Swap = true;
   case ISD::SETGT:  Opc = X86ISD::PCMPGT; break;
   case ISD::SETGE:  Swap = true;
   case ISD::SETLE:  Opc = X86ISD::PCMPGT;
                     Invert = true; break;
   case ISD::SETULT: Swap = true;
   case ISD::SETUGT: Opc = X86ISD::PCMPGT;
                     FlipSigns = true; break;
   case ISD::SETUGE: Swap = true;
   case ISD::SETULE: Opc = X86ISD::PCMPGT;
                     FlipSigns = true; Invert = true; break;
   }
 
   // Special case: Use min/max operations for SETULE/SETUGE
   MVT VET = VT.getVectorElementType();
   bool hasMinMax =
        (Subtarget->hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32))
     || (Subtarget->hasSSE2()  && (VET == MVT::i8));
 
   if (hasMinMax) {
     switch (SetCCOpcode) {
     default: break;
     case ISD::SETULE: Opc = X86ISD::UMIN; MinMax = true; break;
     case ISD::SETUGE: Opc = X86ISD::UMAX; MinMax = true; break;
     }
 
     if (MinMax) { Swap = false; Invert = false; FlipSigns = false; }
   }
 
   bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16);
   if (!MinMax && hasSubus) {
     // As another special case, use PSUBUS[BW] when it's profitable. E.g. for
     // Op0 u<= Op1:
     //   t = psubus Op0, Op1
     //   pcmpeq t, <0..0>
     switch (SetCCOpcode) {
     default: break;
     case ISD::SETULT: {
       // If the comparison is against a constant we can turn this into a
       // setule.  With psubus, setule does not require a swap.  This is
       // beneficial because the constant in the register is no longer
       // destructed as the destination so it can be hoisted out of a loop.
       // Only do this pre-AVX since vpcmp* is no longer destructive.
       if (Subtarget->hasAVX())
         break;
       SDValue ULEOp1 = ChangeVSETULTtoVSETULE(dl, Op1, DAG);
       if (ULEOp1.getNode()) {
         Op1 = ULEOp1;
         Subus = true; Invert = false; Swap = false;
       }
       break;
     }
     // Psubus is better than flip-sign because it requires no inversion.
     case ISD::SETUGE: Subus = true; Invert = false; Swap = true;  break;
     case ISD::SETULE: Subus = true; Invert = false; Swap = false; break;
     }
 
     if (Subus) {
       Opc = X86ISD::SUBUS;
       FlipSigns = false;
     }
   }
 
   if (Swap)
     std::swap(Op0, Op1);
 
   // Check that the operation in question is available (most are plain SSE2,
   // but PCMPGTQ and PCMPEQQ have different requirements).
   if (VT == MVT::v2i64) {
     if (Opc == X86ISD::PCMPGT && !Subtarget->hasSSE42()) {
       assert(Subtarget->hasSSE2() && "Don't know how to lower!");
 
       // First cast everything to the right type.
       Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0);
       Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1);
 
       // Since SSE has no unsigned integer comparisons, we need to flip the sign
       // bits of the inputs before performing those operations. The lower
       // compare is always unsigned.
       SDValue SB;
       if (FlipSigns) {
         SB = DAG.getConstant(0x80000000U, MVT::v4i32);
       } else {
         SDValue Sign = DAG.getConstant(0x80000000U, MVT::i32);
         SDValue Zero = DAG.getConstant(0x00000000U, MVT::i32);
         SB = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
                          Sign, Zero, Sign, Zero);
       }
       Op0 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op0, SB);
       Op1 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op1, SB);
 
       // Emulate PCMPGTQ with (hi1 > hi2) | ((hi1 == hi2) & (lo1 > lo2))
       SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
       SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
 
       // Create masks for only the low parts/high parts of the 64 bit integers.
       static const int MaskHi[] = { 1, 1, 3, 3 };
       static const int MaskLo[] = { 0, 0, 2, 2 };
       SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
       SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
       SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
 
       SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
       Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);
 
       if (Invert)
         Result = DAG.getNOT(dl, Result, MVT::v4i32);
 
       return DAG.getNode(ISD::BITCAST, dl, VT, Result);
     }
 
     if (Opc == X86ISD::PCMPEQ && !Subtarget->hasSSE41()) {
       // If pcmpeqq is missing but pcmpeqd is available synthesize pcmpeqq with
       // pcmpeqd + pshufd + pand.
       assert(Subtarget->hasSSE2() && !FlipSigns && "Don't know how to lower!");
 
       // First cast everything to the right type.
       Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0);
       Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1);
 
       // Do the compare.
       SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
 
       // Make sure the lower and upper halves are both all-ones.
       static const int Mask[] = { 1, 0, 3, 2 };
       SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
       Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
 
       if (Invert)
         Result = DAG.getNOT(dl, Result, MVT::v4i32);
 
       return DAG.getNode(ISD::BITCAST, dl, VT, Result);
     }
   }
 
   // Since SSE has no unsigned integer comparisons, we need to flip the sign
   // bits of the inputs before performing those operations.
   if (FlipSigns) {
     EVT EltVT = VT.getVectorElementType();
     SDValue SB = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), VT);
     Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SB);
     Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SB);
   }
 
   SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
 
   // If the logical-not of the result is required, perform that now.
   if (Invert)
     Result = DAG.getNOT(dl, Result, VT);
 
   if (MinMax)
     Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
 
   if (Subus)
     Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
                          getZeroVector(VT, Subtarget, DAG, dl));
 
   return Result;
 }

static SDValue LowerXALUO	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 15808 of file X86ISelLowering.cpp.

                                                          {
   // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
   // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
   // looks for this combo and may remove the "setcc" instruction if the "setcc"
   // has only one use.
   SDNode *N = Op.getNode();
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
   unsigned BaseOp = 0;
   unsigned Cond = 0;
   SDLoc DL(Op);
   switch (Op.getOpcode()) {
   default: llvm_unreachable("Unknown ovf instruction!");
   case ISD::SADDO:
     // A subtract of one will be selected as a INC. Note that INC doesn't
     // set CF, so we can't do this for UADDO.
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
       if (C->isOne()) {
         BaseOp = X86ISD::INC;
         Cond = X86::COND_O;
         break;
       }
     BaseOp = X86ISD::ADD;
     Cond = X86::COND_O;
     break;
   case ISD::UADDO:
     BaseOp = X86ISD::ADD;
     Cond = X86::COND_B;
     break;
   case ISD::SSUBO:
     // A subtract of one will be selected as a DEC. Note that DEC doesn't
     // set CF, so we can't do this for USUBO.
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
       if (C->isOne()) {
         BaseOp = X86ISD::DEC;
         Cond = X86::COND_O;
         break;
       }
     BaseOp = X86ISD::SUB;
     Cond = X86::COND_O;
     break;
   case ISD::USUBO:
     BaseOp = X86ISD::SUB;
     Cond = X86::COND_B;
     break;
   case ISD::SMULO:
     BaseOp = X86ISD::SMUL;
     Cond = X86::COND_O;
     break;
   case ISD::UMULO: { // i64, i8 = umulo lhs, rhs --> i64, i64, i32 umul lhs,rhs
     SDVTList VTs = DAG.getVTList(N->getValueType(0), N->getValueType(0),
                                  MVT::i32);
     SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);
 
     SDValue SetCC =
       DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
                   DAG.getConstant(X86::COND_O, MVT::i32),
                   SDValue(Sum.getNode(), 2));
 
     return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
   }
   }
 
   // Also sets EFLAGS.
   SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
   SDValue Sum = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
 
   SDValue SetCC =
     DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1),
                 DAG.getConstant(Cond, MVT::i32),
                 SDValue(Sum.getNode(), 1));
 
   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
 }

static SDValue LowerZERO_EXTEND	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 11165 of file X86ISelLowering.cpp.

                                                    {
   SDLoc DL(Op);
   MVT VT = Op.getSimpleValueType();
   SDValue In = Op.getOperand(0);
   MVT SVT = In.getSimpleValueType();
 
   if (VT.is512BitVector() || SVT.getVectorElementType() == MVT::i1)
     return LowerZERO_EXTEND_AVX512(Op, DAG);
 
   if (Subtarget->hasFp256()) {
     SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
     if (Res.getNode())
       return Res;
   }
 
   assert(!VT.is256BitVector() || !SVT.is128BitVector() ||
          VT.getVectorNumElements() != SVT.getVectorNumElements());
   return SDValue();
 }

static SDValue LowerZERO_EXTEND_AVX512	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 11123 of file X86ISelLowering.cpp.

                                                            {
   MVT VT = Op->getSimpleValueType(0);
   SDValue In = Op->getOperand(0);
   MVT InVT = In.getSimpleValueType();
   SDLoc DL(Op);
   unsigned int NumElts = VT.getVectorNumElements();
   if (NumElts != 8 && NumElts != 16)
     return SDValue();
 
   if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1)
     return DAG.getNode(X86ISD::VZEXT, DL, VT, In);
 
   EVT ExtVT = (NumElts == 8)? MVT::v8i64 : MVT::v16i32;
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   // Now we have only mask extension
   assert(InVT.getVectorElementType() == MVT::i1);
   SDValue Cst = DAG.getTargetConstant(1, ExtVT.getScalarType());
   const Constant *C = (dyn_cast<ConstantSDNode>(Cst))->getConstantIntValue();
   SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
   unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
   SDValue Ld = DAG.getLoad(Cst.getValueType(), DL, DAG.getEntryNode(), CP,
                            MachinePointerInfo::getConstantPool(),
                            false, false, false, Alignment);
 
   SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, DL, ExtVT, In, Ld);
   if (VT.is512BitVector())
     return Brcst;
   return DAG.getNode(X86ISD::VTRUNC, DL, VT, Brcst);
 }

static SDValue matchAddSub	(	const BuildVectorSDNode *	BV,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Try to fold a build_vector that performs an 'addsub' into the sequence of 'vadd + vsub + blendi'.

Definition at line 6227 of file X86ISelLowering.cpp.

                                                           {
   SDLoc DL(BV);
   EVT VT = BV->getValueType(0);
   unsigned NumElts = VT.getVectorNumElements();
   SDValue InVec0 = DAG.getUNDEF(VT);
   SDValue InVec1 = DAG.getUNDEF(VT);
 
   assert((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v4f32 ||
           VT == MVT::v2f64) && "build_vector with an invalid type found!");
 
   // Don't try to emit a VSELECT that cannot be lowered into a blend.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
     return SDValue();
 
   // Odd-numbered elements in the input build vector are obtained from
   // adding two integer/float elements.
   // Even-numbered elements in the input build vector are obtained from
   // subtracting two integer/float elements.
   unsigned ExpectedOpcode = ISD::FSUB;
   unsigned NextExpectedOpcode = ISD::FADD;
   bool AddFound = false;
   bool SubFound = false;
 
   for (unsigned i = 0, e = NumElts; i != e; i++) {
     SDValue Op = BV->getOperand(i);
       
     // Skip 'undef' values.
     unsigned Opcode = Op.getOpcode();
     if (Opcode == ISD::UNDEF) {
       std::swap(ExpectedOpcode, NextExpectedOpcode);
       continue;
     }
       
     // Early exit if we found an unexpected opcode.
     if (Opcode != ExpectedOpcode)
       return SDValue();
 
     SDValue Op0 = Op.getOperand(0);
     SDValue Op1 = Op.getOperand(1);
 
     // Try to match the following pattern:
     // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i))
     // Early exit if we cannot match that sequence.
     if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
         Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
         !isa<ConstantSDNode>(Op0.getOperand(1)) ||
         !isa<ConstantSDNode>(Op1.getOperand(1)) ||
         Op0.getOperand(1) != Op1.getOperand(1))
       return SDValue();
 
     unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
     if (I0 != i)
       return SDValue();
 
     // We found a valid add/sub node. Update the information accordingly.
     if (i & 1)
       AddFound = true;
     else
       SubFound = true;
 
     // Update InVec0 and InVec1.
     if (InVec0.getOpcode() == ISD::UNDEF)
       InVec0 = Op0.getOperand(0);
     if (InVec1.getOpcode() == ISD::UNDEF)
       InVec1 = Op1.getOperand(0);
 
     // Make sure that operands in input to each add/sub node always
     // come from a same pair of vectors.
     if (InVec0 != Op0.getOperand(0)) {
       if (ExpectedOpcode == ISD::FSUB)
         return SDValue();
 
       // FADD is commutable. Try to commute the operands
       // and then test again.
       std::swap(Op0, Op1);
       if (InVec0 != Op0.getOperand(0))
         return SDValue();
     }
 
     if (InVec1 != Op1.getOperand(0))
       return SDValue();
 
     // Update the pair of expected opcodes.
     std::swap(ExpectedOpcode, NextExpectedOpcode);
   }
 
   // Don't try to fold this build_vector into a VSELECT if it has
   // too many UNDEF operands.
   if (AddFound && SubFound && InVec0.getOpcode() != ISD::UNDEF &&
       InVec1.getOpcode() != ISD::UNDEF) {
     // Emit a sequence of vector add and sub followed by a VSELECT.
     // The new VSELECT will be lowered into a BLENDI.
     // At ISel stage, we pattern-match the sequence 'add + sub + BLENDI'
     // and emit a single ADDSUB instruction.
     SDValue Sub = DAG.getNode(ExpectedOpcode, DL, VT, InVec0, InVec1);
     SDValue Add = DAG.getNode(NextExpectedOpcode, DL, VT, InVec0, InVec1);
 
     // Construct the VSELECT mask.
     EVT MaskVT = VT.changeVectorElementTypeToInteger();
     EVT SVT = MaskVT.getVectorElementType();
     unsigned SVTBits = SVT.getSizeInBits();
     SmallVector<SDValue, 8> Ops;
 
     for (unsigned i = 0, e = NumElts; i != e; ++i) {
       APInt Value = i & 1 ? APInt::getNullValue(SVTBits) :
                             APInt::getAllOnesValue(SVTBits);
       SDValue Constant = DAG.getConstant(Value, SVT);
       Ops.push_back(Constant);
     }
 
     SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskVT, Ops);
     return DAG.getSelect(DL, VT, Mask, Sub, Add);
   }
   
   return SDValue();
 }

static bool MatchingStackOffset	(	SDValue	Arg,
		unsigned	Offset,
		ISD::ArgFlagsTy	Flags,
		MachineFrameInfo *	MFI,
		const MachineRegisterInfo *	MRI,
		const X86InstrInfo *	TII
	)

static

MatchingStackOffset - Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack.

Definition at line 3104 of file X86ISelLowering.cpp.

                                                   {
   unsigned Bytes = Arg.getValueType().getSizeInBits() / 8;
   int FI = INT_MAX;
   if (Arg.getOpcode() == ISD::CopyFromReg) {
     unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
     if (!TargetRegisterInfo::isVirtualRegister(VR))
       return false;
     MachineInstr *Def = MRI->getVRegDef(VR);
     if (!Def)
       return false;
     if (!Flags.isByVal()) {
       if (!TII->isLoadFromStackSlot(Def, FI))
         return false;
     } else {
       unsigned Opcode = Def->getOpcode();
       if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r) &&
           Def->getOperand(1).isFI()) {
         FI = Def->getOperand(1).getIndex();
         Bytes = Flags.getByValSize();
       } else
         return false;
     }
   } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
     if (Flags.isByVal())
       // ByVal argument is passed in as a pointer but it's now being
       // dereferenced. e.g.
       // define @foo(%struct.X* %A) {
       //   tail call @bar(%struct.X* byval %A)
       // }
       return false;
     SDValue Ptr = Ld->getBasePtr();
     FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
     if (!FINode)
       return false;
     FI = FINode->getIndex();
   } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) {
     FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Arg);
     FI = FINode->getIndex();
     Bytes = Flags.getByValSize();
   } else
     return false;
 
   assert(FI != INT_MAX);
   if (!MFI->isFixedObjectIndex(FI))
     return false;
   return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI);
 }

static std::pair<unsigned, bool> matchIntegerMINMAX	(	SDValue	Cond,
		EVT	VT,
		SDValue	LHS,
		SDValue	RHS,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Matches a VSELECT onto min/max or return 0 if the node doesn't match.

Definition at line 19132 of file X86ISelLowering.cpp.

                                                                      {
   if (!VT.isVector())
     return std::make_pair(0, false);
 
   bool NeedSplit = false;
   switch (VT.getSimpleVT().SimpleTy) {
   default: return std::make_pair(0, false);
   case MVT::v32i8:
   case MVT::v16i16:
   case MVT::v8i32:
     if (!Subtarget->hasAVX2())
       NeedSplit = true;
     if (!Subtarget->hasAVX())
       return std::make_pair(0, false);
     break;
   case MVT::v16i8:
   case MVT::v8i16:
   case MVT::v4i32:
     if (!Subtarget->hasSSE2())
       return std::make_pair(0, false);
   }
 
   // SSE2 has only a small subset of the operations.
   bool hasUnsigned = Subtarget->hasSSE41() ||
                      (Subtarget->hasSSE2() && VT == MVT::v16i8);
   bool hasSigned = Subtarget->hasSSE41() ||
                    (Subtarget->hasSSE2() && VT == MVT::v8i16);
 
   ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
 
   unsigned Opc = 0;
   // Check for x CC y ? x : y.
   if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
       DAG.isEqualTo(RHS, Cond.getOperand(1))) {
     switch (CC) {
     default: break;
     case ISD::SETULT:
     case ISD::SETULE:
       Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
     case ISD::SETUGT:
     case ISD::SETUGE:
       Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
     case ISD::SETLT:
     case ISD::SETLE:
       Opc = hasSigned ? X86ISD::SMIN : 0; break;
     case ISD::SETGT:
     case ISD::SETGE:
       Opc = hasSigned ? X86ISD::SMAX : 0; break;
     }
   // Check for x CC y ? y : x -- a min/max with reversed arms.
   } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
              DAG.isEqualTo(RHS, Cond.getOperand(0))) {
     switch (CC) {
     default: break;
     case ISD::SETULT:
     case ISD::SETULE:
       Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
     case ISD::SETUGT:
     case ISD::SETUGE:
       Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
     case ISD::SETLT:
     case ISD::SETLE:
       Opc = hasSigned ? X86ISD::SMAX : 0; break;
     case ISD::SETGT:
     case ISD::SETGE:
       Opc = hasSigned ? X86ISD::SMIN : 0; break;
     }
   }
 
   return std::make_pair(Opc, NeedSplit);
 }

static SDValue MaterializeSETB	(	SDLoc	DL,
		SDValue	EFLAGS,
		SelectionDAG &	DAG,
		MVT	VT
	)

static

Definition at line 21768 of file X86ISelLowering.cpp.

                                        {
   if (VT == MVT::i8)
     return DAG.getNode(ISD::AND, DL, VT,
                        DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
                                    DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS),
                        DAG.getConstant(1, VT));
   assert (VT == MVT::i1 && "Unexpected type for SECCC node");
   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1,
                      DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
                                  DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS));
 }

static bool MayFoldIntoStore ( SDValue Op )

static

Definition at line 3374 of file X86ISelLowering.cpp.

                                          {
   return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
 }

static bool MayFoldLoad ( SDValue Op )

static

Definition at line 3370 of file X86ISelLowering.cpp.

                                     {
   return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode());
 }

static bool MayFoldVectorLoad ( SDValue V )

static

Definition at line 8892 of file X86ISelLowering.cpp.

                                          {
   while (V.hasOneUse() && V.getOpcode() == ISD::BITCAST)
     V = V.getOperand(0);
 
   if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR)
     V = V.getOperand(0);
   if (V.hasOneUse() && V.getOpcode() == ISD::BUILD_VECTOR &&
       V.getNumOperands() == 2 && V.getOperand(1).getOpcode() == ISD::UNDEF)
     // BUILD_VECTOR (load), undef
     V = V.getOperand(0);
 
   return MayFoldLoad(V);
 }

static SDValue NarrowVectorLoadToElement	(	LoadSDNode *	Load,
		unsigned	Index,
		SelectionDAG &	DAG
	)

static

Definition at line 9009 of file X86ISelLowering.cpp.

                                                             {
   SDLoc dl(Load);
   MVT VT = Load->getSimpleValueType(0);
   MVT EVT = VT.getVectorElementType();
   SDValue Addr = Load->getOperand(1);
   SDValue NewAddr = DAG.getNode(
       ISD::ADD, dl, Addr.getSimpleValueType(), Addr,
       DAG.getConstant(Index * EVT.getStoreSize(), Addr.getSimpleValueType()));
 
   SDValue NewLoad =
       DAG.getLoad(EVT, dl, Load->getChain(), NewAddr,
                   DAG.getMachineFunction().getMachineMemOperand(
                       Load->getMemOperand(), 0, EVT.getStoreSize()));
   return NewLoad;
 }

static void NormalizeMask	(	SmallVectorImpl< int > &	Mask,
		unsigned	NumElems
	)

static

NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements that point to V2 points to its first element.

Definition at line 4956 of file X86ISelLowering.cpp.

                                                                          {
   for (unsigned i = 0; i != NumElems; ++i) {
     if (Mask[i] > (int)NumElems) {
       Mask[i] = NumElems;
     }
   }
 }

static SDValue NormalizeVectorShuffle	(	SDValue	Op,
		const X86Subtarget *	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 9201 of file X86ISelLowering.cpp.

                                                          {
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
   SDValue V1 = Op.getOperand(0);
   SDValue V2 = Op.getOperand(1);
 
   if (isZeroShuffle(SVOp))
     return getZeroVector(VT, Subtarget, DAG, dl);
 
   // Handle splat operations
   if (SVOp->isSplat()) {
     // Use vbroadcast whenever the splat comes from a foldable load
     SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG);
     if (Broadcast.getNode())
       return Broadcast;
   }
 
   // Check integer expanding shuffles.
   SDValue NewOp = LowerVectorIntExtend(Op, Subtarget, DAG);
   if (NewOp.getNode())
     return NewOp;
 
   // If the shuffle can be profitably rewritten as a narrower shuffle, then
   // do it!
   if (VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v16i16 ||
       VT == MVT::v32i8) {
     SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
     if (NewOp.getNode())
       return DAG.getNode(ISD::BITCAST, dl, VT, NewOp);
   } else if (VT.is128BitVector() && Subtarget->hasSSE2()) {
     // FIXME: Figure out a cleaner way to do this.
     if (ISD::isBuildVectorAllZeros(V2.getNode())) {
       SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
       if (NewOp.getNode()) {
         MVT NewVT = NewOp.getSimpleValueType();
         if (isCommutedMOVLMask(cast<ShuffleVectorSDNode>(NewOp)->getMask(),
                                NewVT, true, false))
           return getVZextMovL(VT, NewVT, NewOp.getOperand(0), DAG, Subtarget,
                               dl);
       }
     } else if (ISD::isBuildVectorAllZeros(V1.getNode())) {
       SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
       if (NewOp.getNode()) {
         MVT NewVT = NewOp.getSimpleValueType();
         if (isMOVLMask(cast<ShuffleVectorSDNode>(NewOp)->getMask(), NewVT))
           return getVZextMovL(VT, NewVT, NewOp.getOperand(1), DAG, Subtarget,
                               dl);
       }
     }
   }
   return SDValue();
 }

static SDValue OptimizeConditionalInDecrement	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 21960 of file X86ISelLowering.cpp.

                                                                             {
   SDLoc DL(N);
 
   // Look through ZExts.
   SDValue Ext = N->getOperand(N->getOpcode() == ISD::SUB ? 1 : 0);
   if (Ext.getOpcode() != ISD::ZERO_EXTEND || !Ext.hasOneUse())
     return SDValue();
 
   SDValue SetCC = Ext.getOperand(0);
   if (SetCC.getOpcode() != X86ISD::SETCC || !SetCC.hasOneUse())
     return SDValue();
 
   X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0);
   if (CC != X86::COND_E && CC != X86::COND_NE)
     return SDValue();
 
   SDValue Cmp = SetCC.getOperand(1);
   if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() ||
       !X86::isZeroNode(Cmp.getOperand(1)) ||
       !Cmp.getOperand(0).getValueType().isInteger())
     return SDValue();
 
   SDValue CmpOp0 = Cmp.getOperand(0);
   SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0,
                                DAG.getConstant(1, CmpOp0.getValueType()));
 
   SDValue OtherVal = N->getOperand(N->getOpcode() == ISD::SUB ? 0 : 1);
   if (CC == X86::COND_NE)
     return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB,
                        DL, OtherVal.getValueType(), OtherVal,
                        DAG.getConstant(-1ULL, OtherVal.getValueType()), NewCmp);
   return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC,
                      DL, OtherVal.getValueType(), OtherVal,
                      DAG.getConstant(0, OtherVal.getValueType()), NewCmp);
 }

static SDValue PerformADCCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		X86TargetLowering::DAGCombinerInfo &	DCI
	)

static

Definition at line 21932 of file X86ISelLowering.cpp.

                                                                         {
   // If the LHS and RHS of the ADC node are zero, then it can't overflow and
   // the result is either zero or one (depending on the input carry bit).
   // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1.
   if (X86::isZeroNode(N->getOperand(0)) &&
       X86::isZeroNode(N->getOperand(1)) &&
       // We don't have a good way to replace an EFLAGS use, so only do this when
       // dead right now.
       SDValue(N, 1).use_empty()) {
     SDLoc DL(N);
     EVT VT = N->getValueType(0);
     SDValue CarryOut = DAG.getConstant(0, N->getValueType(1));
     SDValue Res1 = DAG.getNode(ISD::AND, DL, VT,
                                DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
                                            DAG.getConstant(X86::COND_B,MVT::i8),
                                            N->getOperand(2)),
                                DAG.getConstant(1, VT));
     return DCI.CombineTo(N, Res1, CarryOut);
   }
 
   return SDValue();
 }

static SDValue PerformAddCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

PerformADDCombine - Do target-specific dag combines on integer adds.

Definition at line 21997 of file X86ISelLowering.cpp.

                                                                 {
   EVT VT = N->getValueType(0);
   SDValue Op0 = N->getOperand(0);
   SDValue Op1 = N->getOperand(1);
 
   // Try to synthesize horizontal adds from adds of shuffles.
   if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) ||
        (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) &&
       isHorizontalBinOp(Op0, Op1, true))
     return DAG.getNode(X86ISD::HADD, SDLoc(N), VT, Op0, Op1);
 
   return OptimizeConditionalInDecrement(N, DAG);
 }

static SDValue PerformAndCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 20590 of file X86ISelLowering.cpp.

                                                                 {
   EVT VT = N->getValueType(0);
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
   SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget);
   if (R.getNode())
     return R;
 
   // Create BEXTR instructions
   // BEXTR is ((X >> imm) & (2**size-1))
   if (VT == MVT::i32 || VT == MVT::i64) {
     SDValue N0 = N->getOperand(0);
     SDValue N1 = N->getOperand(1);
     SDLoc DL(N);
 
     // Check for BEXTR.
     if ((Subtarget->hasBMI() || Subtarget->hasTBM()) &&
         (N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::SRL)) {
       ConstantSDNode *MaskNode = dyn_cast<ConstantSDNode>(N1);
       ConstantSDNode *ShiftNode = dyn_cast<ConstantSDNode>(N0.getOperand(1));
       if (MaskNode && ShiftNode) {
         uint64_t Mask = MaskNode->getZExtValue();
         uint64_t Shift = ShiftNode->getZExtValue();
         if (isMask_64(Mask)) {
           uint64_t MaskSize = CountPopulation_64(Mask);
           if (Shift + MaskSize <= VT.getSizeInBits())
             return DAG.getNode(X86ISD::BEXTR, DL, VT, N0.getOperand(0),
                                DAG.getConstant(Shift | (MaskSize << 8), VT));
         }
       }
     } // BEXTR
 
     return SDValue();
   }
 
   // Want to form ANDNP nodes:
   // 1) In the hopes of then easily combining them with OR and AND nodes
   //    to form PBLEND/PSIGN.
   // 2) To match ANDN packed intrinsics
   if (VT != MVT::v2i64 && VT != MVT::v4i64)
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDLoc DL(N);
 
   // Check LHS for vnot
   if (N0.getOpcode() == ISD::XOR &&
       //ISD::isBuildVectorAllOnes(N0.getOperand(1).getNode()))
       CanFoldXORWithAllOnes(N0.getOperand(1).getNode()))
     return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1);
 
   // Check RHS for vnot
   if (N1.getOpcode() == ISD::XOR &&
       //ISD::isBuildVectorAllOnes(N1.getOperand(1).getNode()))
       CanFoldXORWithAllOnes(N1.getOperand(1).getNode()))
     return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0);
 
   return SDValue();
 }

static SDValue PerformBrCondCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 21826 of file X86ISelLowering.cpp.

                                                                    {
   SDLoc DL(N);
   SDValue Chain = N->getOperand(0);
   SDValue Dest = N->getOperand(1);
   SDValue EFLAGS = N->getOperand(3);
   X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
 
   SDValue Flags;
 
   Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
   if (Flags.getNode()) {
     SDValue Cond = DAG.getConstant(CC, MVT::i8);
     return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), Chain, Dest, Cond,
                        Flags);
   }
 
   return SDValue();
 }

static SDValue PerformBTCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI
	)

static

Definition at line 21509 of file X86ISelLowering.cpp.

                                                                     {
   // BT ignores high bits in the bit index operand.
   SDValue Op1 = N->getOperand(1);
   if (Op1.hasOneUse()) {
     unsigned BitWidth = Op1.getValueSizeInBits();
     APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
     APInt KnownZero, KnownOne;
     TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                                           !DCI.isBeforeLegalizeOps());
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     if (TLO.ShrinkDemandedConstant(Op1, DemandedMask) ||
         TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO))
       DCI.CommitTargetLoweringOpt(TLO);
   }
   return SDValue();
 }

static SDValue PerformBUILD_VECTORCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 6346 of file X86ISelLowering.cpp.

                                                                          {
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
   unsigned NumElts = VT.getVectorNumElements();
   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(N);
   SDValue InVec0, InVec1;
 
   // Try to match an ADDSUB.
   if ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
       (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) {
     SDValue Value = matchAddSub(BV, DAG, Subtarget);
     if (Value.getNode())
       return Value;
   }
 
   // Try to match horizontal ADD/SUB.
   unsigned NumUndefsLO = 0;
   unsigned NumUndefsHI = 0;
   unsigned Half = NumElts/2;
 
   // Count the number of UNDEF operands in the build_vector in input.
   for (unsigned i = 0, e = Half; i != e; ++i)
     if (BV->getOperand(i)->getOpcode() == ISD::UNDEF)
       NumUndefsLO++;
 
   for (unsigned i = Half, e = NumElts; i != e; ++i)
     if (BV->getOperand(i)->getOpcode() == ISD::UNDEF)
       NumUndefsHI++;
 
   // Early exit if this is either a build_vector of all UNDEFs or all the
   // operands but one are UNDEF.
   if (NumUndefsLO + NumUndefsHI + 1 >= NumElts)
     return SDValue();
 
   if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) {
     // Try to match an SSE3 float HADD/HSUB.
     if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
       return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
     
     if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1))
       return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
   } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) {
     // Try to match an SSSE3 integer HADD/HSUB.
     if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
       return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1);
     
     if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1))
       return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1);
   }
   
   if (!Subtarget->hasAVX())
     return SDValue();
 
   if ((VT == MVT::v8f32 || VT == MVT::v4f64)) {
     // Try to match an AVX horizontal add/sub of packed single/double
     // precision floating point values from 256-bit vectors.
     SDValue InVec2, InVec3;
     if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, Half, InVec0, InVec1) &&
         isHorizontalBinOp(BV, ISD::FADD, DAG, Half, NumElts, InVec2, InVec3) &&
         ((InVec0.getOpcode() == ISD::UNDEF ||
           InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
         ((InVec1.getOpcode() == ISD::UNDEF ||
           InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
       return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
 
     if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, Half, InVec0, InVec1) &&
         isHorizontalBinOp(BV, ISD::FSUB, DAG, Half, NumElts, InVec2, InVec3) &&
         ((InVec0.getOpcode() == ISD::UNDEF ||
           InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
         ((InVec1.getOpcode() == ISD::UNDEF ||
           InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
       return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
   } else if (VT == MVT::v8i32 || VT == MVT::v16i16) {
     // Try to match an AVX2 horizontal add/sub of signed integers.
     SDValue InVec2, InVec3;
     unsigned X86Opcode;
     bool CanFold = true;
 
     if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) &&
         isHorizontalBinOp(BV, ISD::ADD, DAG, Half, NumElts, InVec2, InVec3) &&
         ((InVec0.getOpcode() == ISD::UNDEF ||
           InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
         ((InVec1.getOpcode() == ISD::UNDEF ||
           InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
       X86Opcode = X86ISD::HADD;
     else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, Half, InVec0, InVec1) &&
         isHorizontalBinOp(BV, ISD::SUB, DAG, Half, NumElts, InVec2, InVec3) &&
         ((InVec0.getOpcode() == ISD::UNDEF ||
           InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
         ((InVec1.getOpcode() == ISD::UNDEF ||
           InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
       X86Opcode = X86ISD::HSUB;
     else
       CanFold = false;
 
     if (CanFold) {
       // Fold this build_vector into a single horizontal add/sub.
       // Do this only if the target has AVX2.
       if (Subtarget->hasAVX2())
         return DAG.getNode(X86Opcode, DL, VT, InVec0, InVec1);
  
       // Do not try to expand this build_vector into a pair of horizontal
       // add/sub if we can emit a pair of scalar add/sub.
       if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
         return SDValue();
 
       // Convert this build_vector into a pair of horizontal binop followed by
       // a concat vector.
       bool isUndefLO = NumUndefsLO == Half;
       bool isUndefHI = NumUndefsHI == Half;
       return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, false,
                                    isUndefLO, isUndefHI);
     }
   }
 
   if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
        VT == MVT::v16i16) && Subtarget->hasAVX()) {
     unsigned X86Opcode;
     if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
       X86Opcode = X86ISD::HADD;
     else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1))
       X86Opcode = X86ISD::HSUB;
     else if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
       X86Opcode = X86ISD::FHADD;
     else if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1))
       X86Opcode = X86ISD::FHSUB;
     else
       return SDValue();
 
     // Don't try to expand this build_vector into a pair of horizontal add/sub
     // if we can simply emit a pair of scalar add/sub.
     if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
       return SDValue();
 
     // Convert this build_vector into two horizontal add/sub followed by
     // a concat vector.
     bool isUndefLO = NumUndefsLO == Half;
     bool isUndefHI = NumUndefsHI == Half;
     return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
                                  isUndefLO, isUndefHI);
   }
 
   return SDValue();
 }

static SDValue PerformCMOVCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].

Definition at line 19950 of file X86ISelLowering.cpp.

                                                                  {
   SDLoc DL(N);
 
   // If the flag operand isn't dead, don't touch this CMOV.
   if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty())
     return SDValue();
 
   SDValue FalseOp = N->getOperand(0);
   SDValue TrueOp = N->getOperand(1);
   X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
   SDValue Cond = N->getOperand(3);
 
   if (CC == X86::COND_E || CC == X86::COND_NE) {
     switch (Cond.getOpcode()) {
     default: break;
     case X86ISD::BSR:
     case X86ISD::BSF:
       // If operand of BSR / BSF are proven never zero, then ZF cannot be set.
       if (DAG.isKnownNeverZero(Cond.getOperand(0)))
         return (CC == X86::COND_E) ? FalseOp : TrueOp;
     }
   }
 
   SDValue Flags;
 
   Flags = checkBoolTestSetCCCombine(Cond, CC);
   if (Flags.getNode() &&
       // Extra check as FCMOV only supports a subset of X86 cond.
       (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC))) {
     SDValue Ops[] = { FalseOp, TrueOp,
                       DAG.getConstant(CC, MVT::i8), Flags };
     return DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
   }
 
   // If this is a select between two integer constants, try to do some
   // optimizations.  Note that the operands are ordered the opposite of SELECT
   // operands.
   if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) {
     if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) {
       // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
       // larger than FalseC (the false value).
       if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
         CC = X86::GetOppositeBranchCondition(CC);
         std::swap(TrueC, FalseC);
         std::swap(TrueOp, FalseOp);
       }
 
       // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3.  Likewise for any pow2/0.
       // This is efficient for any integer data type (including i8/i16) and
       // shift amount.
       if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
         Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
                            DAG.getConstant(CC, MVT::i8), Cond);
 
         // Zero extend the condition if needed.
         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
 
         unsigned ShAmt = TrueC->getAPIntValue().logBase2();
         Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
                            DAG.getConstant(ShAmt, MVT::i8));
         if (N->getNumValues() == 2)  // Dead flag value?
           return DCI.CombineTo(N, Cond, SDValue());
         return Cond;
       }
 
       // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.  This is efficient
       // for any integer data type, including i8/i16.
       if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
         Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
                            DAG.getConstant(CC, MVT::i8), Cond);
 
         // Zero extend the condition if needed.
         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
                            FalseC->getValueType(0), Cond);
         Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
                            SDValue(FalseC, 0));
 
         if (N->getNumValues() == 2)  // Dead flag value?
           return DCI.CombineTo(N, Cond, SDValue());
         return Cond;
       }
 
       // Optimize cases that will turn into an LEA instruction.  This requires
       // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
       if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
         uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
         if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
 
         bool isFastMultiplier = false;
         if (Diff < 10) {
           switch ((unsigned char)Diff) {
           default: break;
           case 1:  // result = add base, cond
           case 2:  // result = lea base(    , cond*2)
           case 3:  // result = lea base(cond, cond*2)
           case 4:  // result = lea base(    , cond*4)
           case 5:  // result = lea base(cond, cond*4)
           case 8:  // result = lea base(    , cond*8)
           case 9:  // result = lea base(cond, cond*8)
             isFastMultiplier = true;
             break;
           }
         }
 
         if (isFastMultiplier) {
           APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
           Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
                              DAG.getConstant(CC, MVT::i8), Cond);
           // Zero extend the condition if needed.
           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
                              Cond);
           // Scale the condition by the difference.
           if (Diff != 1)
             Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
                                DAG.getConstant(Diff, Cond.getValueType()));
 
           // Add the base if non-zero.
           if (FalseC->getAPIntValue() != 0)
             Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
                                SDValue(FalseC, 0));
           if (N->getNumValues() == 2)  // Dead flag value?
             return DCI.CombineTo(N, Cond, SDValue());
           return Cond;
         }
       }
     }
   }
 
   // Handle these cases:
   //   (select (x != c), e, c) -> select (x != c), e, x),
   //   (select (x == c), c, e) -> select (x == c), x, e)
   // where the c is an integer constant, and the "select" is the combination
   // of CMOV and CMP.
   //
   // The rationale for this change is that the conditional-move from a constant
   // needs two instructions, however, conditional-move from a register needs
   // only one instruction.
   //
   // CAVEAT: By replacing a constant with a symbolic value, it may obscure
   //  some instruction-combining opportunities. This opt needs to be
   //  postponed as late as possible.
   //
   if (!DCI.isBeforeLegalize() && !DCI.isBeforeLegalizeOps()) {
     // the DCI.xxxx conditions are provided to postpone the optimization as
     // late as possible.
 
     ConstantSDNode *CmpAgainst = nullptr;
     if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
         (CmpAgainst = dyn_cast<ConstantSDNode>(Cond.getOperand(1))) &&
         !isa<ConstantSDNode>(Cond.getOperand(0))) {
 
       if (CC == X86::COND_NE &&
           CmpAgainst == dyn_cast<ConstantSDNode>(FalseOp)) {
         CC = X86::GetOppositeBranchCondition(CC);
         std::swap(TrueOp, FalseOp);
       }
 
       if (CC == X86::COND_E &&
           CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
         SDValue Ops[] = { FalseOp, Cond.getOperand(0),
                           DAG.getConstant(CC, MVT::i8), Cond };
         return DAG.getNode(X86ISD::CMOV, DL, N->getVTList (), Ops);
       }
     }
   }
 
   return SDValue();
 }

static SDValue PerformEXTRACT_VECTOR_ELTCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI
	)

static

PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts to a simple store and scalar loads to extract the elements.

Definition at line 19036 of file X86ISelLowering.cpp.

                                                                              {
   SDValue NewOp = XFormVExtractWithShuffleIntoLoad(N, DAG, DCI);
   if (NewOp.getNode())
     return NewOp;
 
   SDValue InputVector = N->getOperand(0);
 
   // Detect whether we are trying to convert from mmx to i32 and the bitcast
   // from mmx to v2i32 has a single usage.
   if (InputVector.getNode()->getOpcode() == llvm::ISD::BITCAST &&
       InputVector.getNode()->getOperand(0).getValueType() == MVT::x86mmx &&
       InputVector.hasOneUse() && N->getValueType(0) == MVT::i32)
     return DAG.getNode(X86ISD::MMX_MOVD2W, SDLoc(InputVector),
                        N->getValueType(0),
                        InputVector.getNode()->getOperand(0));
 
   // Only operate on vectors of 4 elements, where the alternative shuffling
   // gets to be more expensive.
   if (InputVector.getValueType() != MVT::v4i32)
     return SDValue();
 
   // Check whether every use of InputVector is an EXTRACT_VECTOR_ELT with a
   // single use which is a sign-extend or zero-extend, and all elements are
   // used.
   SmallVector<SDNode *, 4> Uses;
   unsigned ExtractedElements = 0;
   for (SDNode::use_iterator UI = InputVector.getNode()->use_begin(),
        UE = InputVector.getNode()->use_end(); UI != UE; ++UI) {
     if (UI.getUse().getResNo() != InputVector.getResNo())
       return SDValue();
 
     SDNode *Extract = *UI;
     if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
       return SDValue();
 
     if (Extract->getValueType(0) != MVT::i32)
       return SDValue();
     if (!Extract->hasOneUse())
       return SDValue();
     if (Extract->use_begin()->getOpcode() != ISD::SIGN_EXTEND &&
         Extract->use_begin()->getOpcode() != ISD::ZERO_EXTEND)
       return SDValue();
     if (!isa<ConstantSDNode>(Extract->getOperand(1)))
       return SDValue();
 
     // Record which element was extracted.
     ExtractedElements |=
       1 << cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue();
 
     Uses.push_back(Extract);
   }
 
   // If not all the elements were used, this may not be worthwhile.
   if (ExtractedElements != 15)
     return SDValue();
 
   // Ok, we've now decided to do the transformation.
   SDLoc dl(InputVector);
 
   // Store the value to a temporary stack slot.
   SDValue StackPtr = DAG.CreateStackTemporary(InputVector.getValueType());
   SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InputVector, StackPtr,
                             MachinePointerInfo(), false, false, 0);
 
   // Replace each use (extract) with a load of the appropriate element.
   for (SmallVectorImpl<SDNode *>::iterator UI = Uses.begin(),
        UE = Uses.end(); UI != UE; ++UI) {
     SDNode *Extract = *UI;
 
     // cOMpute the element's address.
     SDValue Idx = Extract->getOperand(1);
     unsigned EltSize =
         InputVector.getValueType().getVectorElementType().getSizeInBits()/8;
     uint64_t Offset = EltSize * cast<ConstantSDNode>(Idx)->getZExtValue();
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     SDValue OffsetVal = DAG.getConstant(Offset, TLI.getPointerTy());
 
     SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(),
                                      StackPtr, OffsetVal);
 
     // Load the scalar.
     SDValue LoadScalar = DAG.getLoad(Extract->getValueType(0), dl, Ch,
                                      ScalarAddr, MachinePointerInfo(),
                                      false, false, false, 0);
 
     // Replace the exact with the load.
     DAG.ReplaceAllUsesOfValueWith(SDValue(Extract, 0), LoadScalar);
   }
 
   // The replacement was made in place; don't return anything.
   return SDValue();
 }

static SDValue PerformFADDCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

PerformFADDCombine - Do target-specific dag combines on floating point adds.

Definition at line 21417 of file X86ISelLowering.cpp.

                                                                  {
   EVT VT = N->getValueType(0);
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
 
   // Try to synthesize horizontal adds from adds of shuffles.
   if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
        (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
       isHorizontalBinOp(LHS, RHS, true))
     return DAG.getNode(X86ISD::FHADD, SDLoc(N), VT, LHS, RHS);
   return SDValue();
 }

static SDValue PerformFANDCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes.

Definition at line 21484 of file X86ISelLowering.cpp.

                                                                 {
   // FAND(0.0, x) -> 0.0
   // FAND(x, 0.0) -> 0.0
   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
     if (C->getValueAPF().isPosZero())
       return N->getOperand(0);
   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
     if (C->getValueAPF().isPosZero())
       return N->getOperand(1);
   return SDValue();
 }

static SDValue PerformFANDNCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

PerformFANDNCombine - Do target-specific dag combines on X86ISD::FANDN nodes.

Definition at line 21497 of file X86ISelLowering.cpp.

                                                                  {
   // FANDN(x, 0.0) -> 0.0
   // FANDN(0.0, x) -> x
   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
     if (C->getValueAPF().isPosZero())
       return N->getOperand(1);
   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
     if (C->getValueAPF().isPosZero())
       return N->getOperand(1);
   return SDValue();
 }

static SDValue PerformFMACombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 21595 of file X86ISelLowering.cpp.

                                                                 {
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
 
   // Let legalize expand this if it isn't a legal type yet.
   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
     return SDValue();
 
   EVT ScalarVT = VT.getScalarType();
   if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||
       (!Subtarget->hasFMA() && !Subtarget->hasFMA4()))
     return SDValue();
 
   SDValue A = N->getOperand(0);
   SDValue B = N->getOperand(1);
   SDValue C = N->getOperand(2);
 
   bool NegA = (A.getOpcode() == ISD::FNEG);
   bool NegB = (B.getOpcode() == ISD::FNEG);
   bool NegC = (C.getOpcode() == ISD::FNEG);
 
   // Negative multiplication when NegA xor NegB
   bool NegMul = (NegA != NegB);
   if (NegA)
     A = A.getOperand(0);
   if (NegB)
     B = B.getOperand(0);
   if (NegC)
     C = C.getOperand(0);
 
   unsigned Opcode;
   if (!NegMul)
     Opcode = (!NegC) ? X86ISD::FMADD : X86ISD::FMSUB;
   else
     Opcode = (!NegC) ? X86ISD::FNMADD : X86ISD::FNMSUB;
 
   return DAG.getNode(Opcode, dl, VT, A, B, C);
 }

static SDValue PerformFMinFMaxCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

PerformFMinFMaxCombine - Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.

Definition at line 21463 of file X86ISelLowering.cpp.

                                                                     {
   assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
 
   // Only perform optimizations if UnsafeMath is used.
   if (!DAG.getTarget().Options.UnsafeFPMath)
     return SDValue();
 
   // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes
   // into FMINC and FMAXC, which are Commutative operations.
   unsigned NewOp = 0;
   switch (N->getOpcode()) {
     default: llvm_unreachable("unknown opcode");
     case X86ISD::FMIN:  NewOp = X86ISD::FMINC; break;
     case X86ISD::FMAX:  NewOp = X86ISD::FMAXC; break;
   }
 
   return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
                      N->getOperand(0), N->getOperand(1));
 }

static SDValue PerformFORCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.

Definition at line 21448 of file X86ISelLowering.cpp.

                                                                {
   assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
   // F[X]OR(0.0, x) -> x
   // F[X]OR(x, 0.0) -> x
   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
     if (C->getValueAPF().isPosZero())
       return N->getOperand(1);
   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
     if (C->getValueAPF().isPosZero())
       return N->getOperand(0);
   return SDValue();
 }

static SDValue PerformFSUBCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

PerformFSUBCombine - Do target-specific dag combines on floating point subs.

Definition at line 21432 of file X86ISelLowering.cpp.

                                                                  {
   EVT VT = N->getValueType(0);
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
 
   // Try to synthesize horizontal subs from subs of shuffles.
   if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
        (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
       isHorizontalBinOp(LHS, RHS, false))
     return DAG.getNode(X86ISD::FHSUB, SDLoc(N), VT, LHS, RHS);
   return SDValue();
 }

static SDValue PerformINSERTPSCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 21738 of file X86ISelLowering.cpp.

                                                                      {
   SDLoc dl(N);
   MVT VT = N->getOperand(1)->getSimpleValueType(0);
   assert((VT == MVT::v4f32 || VT == MVT::v4i32) &&
          "X86insertps is only defined for v4x32");
 
   SDValue Ld = N->getOperand(1);
   if (MayFoldLoad(Ld)) {
     // Extract the countS bits from the immediate so we can get the proper
     // address when narrowing the vector load to a specific element.
     // When the second source op is a memory address, interps doesn't use
     // countS and just gets an f32 from that address.
     unsigned DestIndex =
         cast<ConstantSDNode>(N->getOperand(2))->getZExtValue() >> 6;
     Ld = NarrowVectorLoadToElement(cast<LoadSDNode>(Ld), DestIndex, DAG);
   } else
     return SDValue();
 
   // Create this as a scalar to vector to match the instruction pattern.
   SDValue LoadScalarToVector = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Ld);
   // countS bits are ignored when loading from memory on insertps, which
   // means we don't need to explicitly set them to 0.
   return DAG.getNode(X86ISD::INSERTPS, dl, VT, N->getOperand(0),
                      LoadScalarToVector, N->getOperand(2));
 }

static SDValue performIntegerAbsCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 20816 of file X86ISelLowering.cpp.

                                                                       {
   EVT VT = N->getValueType(0);
 
   // Since X86 does not have CMOV for 8-bit integer, we don't convert
   // 8-bit integer abs to NEG and CMOV.
   if (VT.isInteger() && VT.getSizeInBits() == 8)
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDLoc DL(N);
 
   // Check pattern of XOR(ADD(X,Y), Y) where Y is SRA(X, size(X)-1)
   // and change it to SUB and CMOV.
   if (VT.isInteger() && N->getOpcode() == ISD::XOR &&
       N0.getOpcode() == ISD::ADD &&
       N0.getOperand(1) == N1 &&
       N1.getOpcode() == ISD::SRA &&
       N1.getOperand(0) == N0.getOperand(0))
     if (ConstantSDNode *Y1C = dyn_cast<ConstantSDNode>(N1.getOperand(1)))
       if (Y1C->getAPIntValue() == VT.getSizeInBits()-1) {
         // Generate SUB & CMOV.
         SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
                                   DAG.getConstant(0, VT), N0.getOperand(0));
 
         SDValue Ops[] = { N0.getOperand(0), Neg,
                           DAG.getConstant(X86::COND_GE, MVT::i8),
                           SDValue(Neg.getNode(), 1) };
         return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops);
       }
   return SDValue();
 }

static SDValue PerformINTRINSIC_WO_CHAINCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 20121 of file X86ISelLowering.cpp.

                                                                                {
   unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
   switch (IntNo) {
   default: return SDValue();
   // SSE/AVX/AVX2 blend intrinsics.
   case Intrinsic::x86_avx2_pblendvb:
   case Intrinsic::x86_avx2_pblendw:
   case Intrinsic::x86_avx2_pblendd_128:
   case Intrinsic::x86_avx2_pblendd_256:
     // Don't try to simplify this intrinsic if we don't have AVX2.
     if (!Subtarget->hasAVX2())
       return SDValue();
     // FALL-THROUGH
   case Intrinsic::x86_avx_blend_pd_256:
   case Intrinsic::x86_avx_blend_ps_256:
   case Intrinsic::x86_avx_blendv_pd_256:
   case Intrinsic::x86_avx_blendv_ps_256:
     // Don't try to simplify this intrinsic if we don't have AVX.
     if (!Subtarget->hasAVX())
       return SDValue();
     // FALL-THROUGH
   case Intrinsic::x86_sse41_pblendw:
   case Intrinsic::x86_sse41_blendpd:
   case Intrinsic::x86_sse41_blendps:
   case Intrinsic::x86_sse41_blendvps:
   case Intrinsic::x86_sse41_blendvpd:
   case Intrinsic::x86_sse41_pblendvb: {
     SDValue Op0 = N->getOperand(1);
     SDValue Op1 = N->getOperand(2);
     SDValue Mask = N->getOperand(3);
 
     // Don't try to simplify this intrinsic if we don't have SSE4.1.
     if (!Subtarget->hasSSE41())
       return SDValue();
 
     // fold (blend A, A, Mask) -> A
     if (Op0 == Op1)
       return Op0;
     // fold (blend A, B, allZeros) -> A
     if (ISD::isBuildVectorAllZeros(Mask.getNode()))
       return Op0;
     // fold (blend A, B, allOnes) -> B
     if (ISD::isBuildVectorAllOnes(Mask.getNode()))
       return Op1;
     
     // Simplify the case where the mask is a constant i32 value.
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Mask)) {
       if (C->isNullValue())
         return Op0;
       if (C->isAllOnesValue())
         return Op1;
     }
 
     return SDValue();
   }
 
   // Packed SSE2/AVX2 arithmetic shift immediate intrinsics.
   case Intrinsic::x86_sse2_psrai_w:
   case Intrinsic::x86_sse2_psrai_d:
   case Intrinsic::x86_avx2_psrai_w:
   case Intrinsic::x86_avx2_psrai_d:
   case Intrinsic::x86_sse2_psra_w:
   case Intrinsic::x86_sse2_psra_d:
   case Intrinsic::x86_avx2_psra_w:
   case Intrinsic::x86_avx2_psra_d: {
     SDValue Op0 = N->getOperand(1);
     SDValue Op1 = N->getOperand(2);
     EVT VT = Op0.getValueType();
     assert(VT.isVector() && "Expected a vector type!");
 
     if (isa<BuildVectorSDNode>(Op1))
       Op1 = Op1.getOperand(0);
 
     if (!isa<ConstantSDNode>(Op1))
       return SDValue();
 
     EVT SVT = VT.getVectorElementType();
     unsigned SVTBits = SVT.getSizeInBits();
 
     ConstantSDNode *CND = cast<ConstantSDNode>(Op1);
     const APInt &C = APInt(SVTBits, CND->getAPIntValue().getZExtValue());
     uint64_t ShAmt = C.getZExtValue();
 
     // Don't try to convert this shift into a ISD::SRA if the shift
     // count is bigger than or equal to the element size.
     if (ShAmt >= SVTBits)
       return SDValue();
 
     // Trivial case: if the shift count is zero, then fold this
     // into the first operand.
     if (ShAmt == 0)
       return Op0;
 
     // Replace this packed shift intrinsic with a target independent
     // shift dag node.
     SDValue Splat = DAG.getConstant(C, VT);
     return DAG.getNode(ISD::SRA, SDLoc(N), VT, Op0, Splat);
   }
   }
 }

static SDValue PerformISDSETCCCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 21683 of file X86ISelLowering.cpp.

                                                                      {
   ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
 
   if ((CC == ISD::SETNE || CC == ISD::SETEQ) && LHS.getOpcode() == ISD::SUB)
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS.getOperand(0)))
       if (C->getAPIntValue() == 0 && LHS.hasOneUse()) {
         SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N),
                                    LHS.getValueType(), RHS, LHS.getOperand(1));
         return DAG.getSetCC(SDLoc(N), N->getValueType(0),
                             addV, DAG.getConstant(0, addV.getValueType()), CC);
       }
   if ((CC == ISD::SETNE || CC == ISD::SETEQ) && RHS.getOpcode() == ISD::SUB)
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS.getOperand(0)))
       if (C->getAPIntValue() == 0 && RHS.hasOneUse()) {
         SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N),
                                    RHS.getValueType(), LHS, RHS.getOperand(1));
         return DAG.getSetCC(SDLoc(N), N->getValueType(0),
                             addV, DAG.getConstant(0, addV.getValueType()), CC);
       }
 
   if (VT.getScalarType() == MVT::i1) {
     bool IsSEXT0 = (LHS.getOpcode() == ISD::SIGN_EXTEND) &&
       (LHS.getOperand(0).getValueType().getScalarType() ==  MVT::i1);
     bool IsVZero0 = ISD::isBuildVectorAllZeros(LHS.getNode());
     if (!IsSEXT0 && !IsVZero0)
       return SDValue();
     bool IsSEXT1 = (RHS.getOpcode() == ISD::SIGN_EXTEND) &&
       (RHS.getOperand(0).getValueType().getScalarType() ==  MVT::i1);
     bool IsVZero1 = ISD::isBuildVectorAllZeros(RHS.getNode());
 
     if (!IsSEXT1 && !IsVZero1)
       return SDValue();
 
     if (IsSEXT0 && IsVZero1) {
       assert(VT == LHS.getOperand(0).getValueType() && "Uexpected operand type");
       if (CC == ISD::SETEQ)
         return DAG.getNOT(DL, LHS.getOperand(0), VT);
       return LHS.getOperand(0);
     }
     if (IsSEXT1 && IsVZero0) {
       assert(VT == RHS.getOperand(0).getValueType() && "Uexpected operand type");
       if (CC == ISD::SETEQ)
         return DAG.getNOT(DL, RHS.getOperand(0), VT);
       return RHS.getOperand(0);
     }
   }
 
   return SDValue();
 }

static SDValue PerformLOADCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

PerformLOADCombine - Do target-specific dag combines on LOAD nodes.

Definition at line 20866 of file X86ISelLowering.cpp.

                                                                  {
   LoadSDNode *Ld = cast<LoadSDNode>(N);
   EVT RegVT = Ld->getValueType(0);
   EVT MemVT = Ld->getMemoryVT();
   SDLoc dl(Ld);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   unsigned RegSz = RegVT.getSizeInBits();
 
   // On Sandybridge unaligned 256bit loads are inefficient.
   ISD::LoadExtType Ext = Ld->getExtensionType();
   unsigned Alignment = Ld->getAlignment();
   bool IsAligned = Alignment == 0 || Alignment >= MemVT.getSizeInBits()/8;
   if (RegVT.is256BitVector() && !Subtarget->hasInt256() &&
       !DCI.isBeforeLegalizeOps() && !IsAligned && Ext == ISD::NON_EXTLOAD) {
     unsigned NumElems = RegVT.getVectorNumElements();
     if (NumElems < 2)
       return SDValue();
 
     SDValue Ptr = Ld->getBasePtr();
     SDValue Increment = DAG.getConstant(16, TLI.getPointerTy());
 
     EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
                                   NumElems/2);
     SDValue Load1 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr,
                                 Ld->getPointerInfo(), Ld->isVolatile(),
                                 Ld->isNonTemporal(), Ld->isInvariant(),
                                 Alignment);
     Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
     SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr,
                                 Ld->getPointerInfo(), Ld->isVolatile(),
                                 Ld->isNonTemporal(), Ld->isInvariant(),
                                 std::min(16U, Alignment));
     SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                              Load1.getValue(1),
                              Load2.getValue(1));
 
     SDValue NewVec = DAG.getUNDEF(RegVT);
     NewVec = Insert128BitVector(NewVec, Load1, 0, DAG, dl);
     NewVec = Insert128BitVector(NewVec, Load2, NumElems/2, DAG, dl);
     return DCI.CombineTo(N, NewVec, TF, true);
   }
 
   // If this is a vector EXT Load then attempt to optimize it using a
   // shuffle. If SSSE3 is not available we may emit an illegal shuffle but the
   // expansion is still better than scalar code.
   // We generate X86ISD::VSEXT for SEXTLOADs if it's available, otherwise we'll
   // emit a shuffle and a arithmetic shift.
   // TODO: It is possible to support ZExt by zeroing the undef values
   // during the shuffle phase or after the shuffle.
   if (RegVT.isVector() && RegVT.isInteger() && Subtarget->hasSSE2() &&
       (Ext == ISD::EXTLOAD || Ext == ISD::SEXTLOAD)) {
     assert(MemVT != RegVT && "Cannot extend to the same type");
     assert(MemVT.isVector() && "Must load a vector from memory");
 
     unsigned NumElems = RegVT.getVectorNumElements();
     unsigned MemSz = MemVT.getSizeInBits();
     assert(RegSz > MemSz && "Register size must be greater than the mem size");
 
     if (Ext == ISD::SEXTLOAD && RegSz == 256 && !Subtarget->hasInt256())
       return SDValue();
 
     // All sizes must be a power of two.
     if (!isPowerOf2_32(RegSz * MemSz * NumElems))
       return SDValue();
 
     // Attempt to load the original value using scalar loads.
     // Find the largest scalar type that divides the total loaded size.
     MVT SclrLoadTy = MVT::i8;
     for (unsigned tp = MVT::FIRST_INTEGER_VALUETYPE;
          tp < MVT::LAST_INTEGER_VALUETYPE; ++tp) {
       MVT Tp = (MVT::SimpleValueType)tp;
       if (TLI.isTypeLegal(Tp) && ((MemSz % Tp.getSizeInBits()) == 0)) {
         SclrLoadTy = Tp;
       }
     }
 
     // On 32bit systems, we can't save 64bit integers. Try bitcasting to F64.
     if (TLI.isTypeLegal(MVT::f64) && SclrLoadTy.getSizeInBits() < 64 &&
         (64 <= MemSz))
       SclrLoadTy = MVT::f64;
 
     // Calculate the number of scalar loads that we need to perform
     // in order to load our vector from memory.
     unsigned NumLoads = MemSz / SclrLoadTy.getSizeInBits();
     if (Ext == ISD::SEXTLOAD && NumLoads > 1)
       return SDValue();
 
     unsigned loadRegZize = RegSz;
     if (Ext == ISD::SEXTLOAD && RegSz == 256)
       loadRegZize /= 2;
 
     // Represent our vector as a sequence of elements which are the
     // largest scalar that we can load.
     EVT LoadUnitVecVT = EVT::getVectorVT(*DAG.getContext(), SclrLoadTy,
       loadRegZize/SclrLoadTy.getSizeInBits());
 
     // Represent the data using the same element type that is stored in
     // memory. In practice, we ''widen'' MemVT.
     EVT WideVecVT =
           EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
                        loadRegZize/MemVT.getScalarType().getSizeInBits());
 
     assert(WideVecVT.getSizeInBits() == LoadUnitVecVT.getSizeInBits() &&
       "Invalid vector type");
 
     // We can't shuffle using an illegal type.
     if (!TLI.isTypeLegal(WideVecVT))
       return SDValue();
 
     SmallVector<SDValue, 8> Chains;
     SDValue Ptr = Ld->getBasePtr();
     SDValue Increment = DAG.getConstant(SclrLoadTy.getSizeInBits()/8,
                                         TLI.getPointerTy());
     SDValue Res = DAG.getUNDEF(LoadUnitVecVT);
 
     for (unsigned i = 0; i < NumLoads; ++i) {
       // Perform a single load.
       SDValue ScalarLoad = DAG.getLoad(SclrLoadTy, dl, Ld->getChain(),
                                        Ptr, Ld->getPointerInfo(),
                                        Ld->isVolatile(), Ld->isNonTemporal(),
                                        Ld->isInvariant(), Ld->getAlignment());
       Chains.push_back(ScalarLoad.getValue(1));
       // Create the first element type using SCALAR_TO_VECTOR in order to avoid
       // another round of DAGCombining.
       if (i == 0)
         Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoadUnitVecVT, ScalarLoad);
       else
         Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, LoadUnitVecVT, Res,
                           ScalarLoad, DAG.getIntPtrConstant(i));
 
       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
     }
 
     SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
 
     // Bitcast the loaded value to a vector of the original element type, in
     // the size of the target vector type.
     SDValue SlicedVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Res);
     unsigned SizeRatio = RegSz/MemSz;
 
     if (Ext == ISD::SEXTLOAD) {
       // If we have SSE4.1 we can directly emit a VSEXT node.
       if (Subtarget->hasSSE41()) {
         SDValue Sext = DAG.getNode(X86ISD::VSEXT, dl, RegVT, SlicedVec);
         return DCI.CombineTo(N, Sext, TF, true);
       }
 
       // Otherwise we'll shuffle the small elements in the high bits of the
       // larger type and perform an arithmetic shift. If the shift is not legal
       // it's better to scalarize.
       if (!TLI.isOperationLegalOrCustom(ISD::SRA, RegVT))
         return SDValue();
 
       // Redistribute the loaded elements into the different locations.
       SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
       for (unsigned i = 0; i != NumElems; ++i)
         ShuffleVec[i*SizeRatio + SizeRatio-1] = i;
 
       SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec,
                                            DAG.getUNDEF(WideVecVT),
                                            &ShuffleVec[0]);
 
       Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
 
       // Build the arithmetic shift.
       unsigned Amt = RegVT.getVectorElementType().getSizeInBits() -
                      MemVT.getVectorElementType().getSizeInBits();
       Shuff = DAG.getNode(ISD::SRA, dl, RegVT, Shuff,
                           DAG.getConstant(Amt, RegVT));
 
       return DCI.CombineTo(N, Shuff, TF, true);
     }
 
     // Redistribute the loaded elements into the different locations.
     SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
     for (unsigned i = 0; i != NumElems; ++i)
       ShuffleVec[i*SizeRatio] = i;
 
     SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec,
                                          DAG.getUNDEF(WideVecVT),
                                          &ShuffleVec[0]);
 
     // Bitcast to the requested type.
     Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
     // Replace the original load with the new sequence
     // and return the new chain.
     return DCI.CombineTo(N, Shuff, TF, true);
   }
 
   return SDValue();
 }

static SDValue PerformMulCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI
	)

static

PerformMulCombine - Optimize a single multiply with constant into two in order to implement it with two cheaper instructions, e.g. LEA + SHL, LEA + LEA.

Definition at line 20226 of file X86ISelLowering.cpp.

                                                                      {
   if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
     return SDValue();
 
   EVT VT = N->getValueType(0);
   if (VT != MVT::i64)
     return SDValue();
 
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
   if (!C)
     return SDValue();
   uint64_t MulAmt = C->getZExtValue();
   if (isPowerOf2_64(MulAmt) || MulAmt == 3 || MulAmt == 5 || MulAmt == 9)
     return SDValue();
 
   uint64_t MulAmt1 = 0;
   uint64_t MulAmt2 = 0;
   if ((MulAmt % 9) == 0) {
     MulAmt1 = 9;
     MulAmt2 = MulAmt / 9;
   } else if ((MulAmt % 5) == 0) {
     MulAmt1 = 5;
     MulAmt2 = MulAmt / 5;
   } else if ((MulAmt % 3) == 0) {
     MulAmt1 = 3;
     MulAmt2 = MulAmt / 3;
   }
   if (MulAmt2 &&
       (isPowerOf2_64(MulAmt2) || MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)){
     SDLoc DL(N);
 
     if (isPowerOf2_64(MulAmt2) &&
         !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD))
       // If second multiplifer is pow2, issue it first. We want the multiply by
       // 3, 5, or 9 to be folded into the addressing mode unless the lone use
       // is an add.
       std::swap(MulAmt1, MulAmt2);
 
     SDValue NewMul;
     if (isPowerOf2_64(MulAmt1))
       NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
                            DAG.getConstant(Log2_64(MulAmt1), MVT::i8));
     else
       NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
                            DAG.getConstant(MulAmt1, VT));
 
     if (isPowerOf2_64(MulAmt2))
       NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
                            DAG.getConstant(Log2_64(MulAmt2), MVT::i8));
     else
       NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
                            DAG.getConstant(MulAmt2, VT));
 
     // Do not add new nodes to DAG combiner worklist.
     DCI.CombineTo(N, NewMul, false);
   }
   return SDValue();
 }

static SDValue PerformOrCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 20654 of file X86ISelLowering.cpp.

                                                                {
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
   SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget);
   if (R.getNode())
     return R;
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
 
   // look for psign/blend
   if (VT == MVT::v2i64 || VT == MVT::v4i64) {
     if (!Subtarget->hasSSSE3() ||
         (VT == MVT::v4i64 && !Subtarget->hasInt256()))
       return SDValue();
 
     // Canonicalize pandn to RHS
     if (N0.getOpcode() == X86ISD::ANDNP)
       std::swap(N0, N1);
     // or (and (m, y), (pandn m, x))
     if (N0.getOpcode() == ISD::AND && N1.getOpcode() == X86ISD::ANDNP) {
       SDValue Mask = N1.getOperand(0);
       SDValue X    = N1.getOperand(1);
       SDValue Y;
       if (N0.getOperand(0) == Mask)
         Y = N0.getOperand(1);
       if (N0.getOperand(1) == Mask)
         Y = N0.getOperand(0);
 
       // Check to see if the mask appeared in both the AND and ANDNP and
       if (!Y.getNode())
         return SDValue();
 
       // Validate that X, Y, and Mask are BIT_CONVERTS, and see through them.
       // Look through mask bitcast.
       if (Mask.getOpcode() == ISD::BITCAST)
         Mask = Mask.getOperand(0);
       if (X.getOpcode() == ISD::BITCAST)
         X = X.getOperand(0);
       if (Y.getOpcode() == ISD::BITCAST)
         Y = Y.getOperand(0);
 
       EVT MaskVT = Mask.getValueType();
 
       // Validate that the Mask operand is a vector sra node.
       // FIXME: what to do for bytes, since there is a psignb/pblendvb, but
       // there is no psrai.b
       unsigned EltBits = MaskVT.getVectorElementType().getSizeInBits();
       unsigned SraAmt = ~0;
       if (Mask.getOpcode() == ISD::SRA) {
         if (auto *AmtBV = dyn_cast<BuildVectorSDNode>(Mask.getOperand(1)))
           if (auto *AmtConst = AmtBV->getConstantSplatNode())
             SraAmt = AmtConst->getZExtValue();
       } else if (Mask.getOpcode() == X86ISD::VSRAI) {
         SDValue SraC = Mask.getOperand(1);
         SraAmt  = cast<ConstantSDNode>(SraC)->getZExtValue();
       }
       if ((SraAmt + 1) != EltBits)
         return SDValue();
 
       SDLoc DL(N);
 
       // Now we know we at least have a plendvb with the mask val.  See if
       // we can form a psignb/w/d.
       // psign = x.type == y.type == mask.type && y = sub(0, x);
       if (Y.getOpcode() == ISD::SUB && Y.getOperand(1) == X &&
           ISD::isBuildVectorAllZeros(Y.getOperand(0).getNode()) &&
           X.getValueType() == MaskVT && Y.getValueType() == MaskVT) {
         assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
                "Unsupported VT for PSIGN");
         Mask = DAG.getNode(X86ISD::PSIGN, DL, MaskVT, X, Mask.getOperand(0));
         return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
       }
       // PBLENDVB only available on SSE 4.1
       if (!Subtarget->hasSSE41())
         return SDValue();
 
       EVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8;
 
       X = DAG.getNode(ISD::BITCAST, DL, BlendVT, X);
       Y = DAG.getNode(ISD::BITCAST, DL, BlendVT, Y);
       Mask = DAG.getNode(ISD::BITCAST, DL, BlendVT, Mask);
       Mask = DAG.getNode(ISD::VSELECT, DL, BlendVT, Mask, Y, X);
       return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
     }
   }
 
   if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
     return SDValue();
 
   // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c)
   MachineFunction &MF = DAG.getMachineFunction();
   bool OptForSize = MF.getFunction()->getAttributes().
     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
 
   // SHLD/SHRD instructions have lower register pressure, but on some
   // platforms they have higher latency than the equivalent
   // series of shifts/or that would otherwise be generated.
   // Don't fold (or (x << c) | (y >> (64 - c))) if SHLD/SHRD instructions
   // have higher latencies and we are not optimizing for size.
   if (!OptForSize && Subtarget->isSHLDSlow())
     return SDValue();
 
   if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
     std::swap(N0, N1);
   if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
     return SDValue();
   if (!N0.hasOneUse() || !N1.hasOneUse())
     return SDValue();
 
   SDValue ShAmt0 = N0.getOperand(1);
   if (ShAmt0.getValueType() != MVT::i8)
     return SDValue();
   SDValue ShAmt1 = N1.getOperand(1);
   if (ShAmt1.getValueType() != MVT::i8)
     return SDValue();
   if (ShAmt0.getOpcode() == ISD::TRUNCATE)
     ShAmt0 = ShAmt0.getOperand(0);
   if (ShAmt1.getOpcode() == ISD::TRUNCATE)
     ShAmt1 = ShAmt1.getOperand(0);
 
   SDLoc DL(N);
   unsigned Opc = X86ISD::SHLD;
   SDValue Op0 = N0.getOperand(0);
   SDValue Op1 = N1.getOperand(0);
   if (ShAmt0.getOpcode() == ISD::SUB) {
     Opc = X86ISD::SHRD;
     std::swap(Op0, Op1);
     std::swap(ShAmt0, ShAmt1);
   }
 
   unsigned Bits = VT.getSizeInBits();
   if (ShAmt1.getOpcode() == ISD::SUB) {
     SDValue Sum = ShAmt1.getOperand(0);
     if (ConstantSDNode *SumC = dyn_cast<ConstantSDNode>(Sum)) {
       SDValue ShAmt1Op1 = ShAmt1.getOperand(1);
       if (ShAmt1Op1.getNode()->getOpcode() == ISD::TRUNCATE)
         ShAmt1Op1 = ShAmt1Op1.getOperand(0);
       if (SumC->getSExtValue() == Bits && ShAmt1Op1 == ShAmt0)
         return DAG.getNode(Opc, DL, VT,
                            Op0, Op1,
                            DAG.getNode(ISD::TRUNCATE, DL,
                                        MVT::i8, ShAmt0));
     }
   } else if (ConstantSDNode *ShAmt1C = dyn_cast<ConstantSDNode>(ShAmt1)) {
     ConstantSDNode *ShAmt0C = dyn_cast<ConstantSDNode>(ShAmt0);
     if (ShAmt0C &&
         ShAmt0C->getSExtValue() + ShAmt1C->getSExtValue() == Bits)
       return DAG.getNode(Opc, DL, VT,
                          N0.getOperand(0), N1.getOperand(0),
                          DAG.getNode(ISD::TRUNCATE, DL,
                                        MVT::i8, ShAmt0));
   }
 
   return SDValue();
 }

static SDValue PerformSELECTCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT nodes.

Definition at line 19252 of file X86ISelLowering.cpp.

                                                                    {
   SDLoc DL(N);
   SDValue Cond = N->getOperand(0);
   // Get the LHS/RHS of the select.
   SDValue LHS = N->getOperand(1);
   SDValue RHS = N->getOperand(2);
   EVT VT = LHS.getValueType();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // If we have SSE[12] support, try to form min/max nodes. SSE min/max
   // instructions match the semantics of the common C idiom x<y?x:y but not
   // x<=y?x:y, because of how they handle negative zero (which can be
   // ignored in unsafe-math mode).
   if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
       VT != MVT::f80 && TLI.isTypeLegal(VT) &&
       (Subtarget->hasSSE2() ||
        (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) {
     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
 
     unsigned Opcode = 0;
     // Check for x CC y ? x : y.
     if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
         DAG.isEqualTo(RHS, Cond.getOperand(1))) {
       switch (CC) {
       default: break;
       case ISD::SETULT:
         // Converting this to a min would handle NaNs incorrectly, and swapping
         // the operands would cause it to handle comparisons between positive
         // and negative zero incorrectly.
         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
           if (!DAG.getTarget().Options.UnsafeFPMath &&
               !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
             break;
           std::swap(LHS, RHS);
         }
         Opcode = X86ISD::FMIN;
         break;
       case ISD::SETOLE:
         // Converting this to a min would handle comparisons between positive
         // and negative zero incorrectly.
         if (!DAG.getTarget().Options.UnsafeFPMath &&
             !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS))
           break;
         Opcode = X86ISD::FMIN;
         break;
       case ISD::SETULE:
         // Converting this to a min would handle both negative zeros and NaNs
         // incorrectly, but we can swap the operands to fix both.
         std::swap(LHS, RHS);
       case ISD::SETOLT:
       case ISD::SETLT:
       case ISD::SETLE:
         Opcode = X86ISD::FMIN;
         break;
 
       case ISD::SETOGE:
         // Converting this to a max would handle comparisons between positive
         // and negative zero incorrectly.
         if (!DAG.getTarget().Options.UnsafeFPMath &&
             !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS))
           break;
         Opcode = X86ISD::FMAX;
         break;
       case ISD::SETUGT:
         // Converting this to a max would handle NaNs incorrectly, and swapping
         // the operands would cause it to handle comparisons between positive
         // and negative zero incorrectly.
         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
           if (!DAG.getTarget().Options.UnsafeFPMath &&
               !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
             break;
           std::swap(LHS, RHS);
         }
         Opcode = X86ISD::FMAX;
         break;
       case ISD::SETUGE:
         // Converting this to a max would handle both negative zeros and NaNs
         // incorrectly, but we can swap the operands to fix both.
         std::swap(LHS, RHS);
       case ISD::SETOGT:
       case ISD::SETGT:
       case ISD::SETGE:
         Opcode = X86ISD::FMAX;
         break;
       }
     // Check for x CC y ? y : x -- a min/max with reversed arms.
     } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
                DAG.isEqualTo(RHS, Cond.getOperand(0))) {
       switch (CC) {
       default: break;
       case ISD::SETOGE:
         // Converting this to a min would handle comparisons between positive
         // and negative zero incorrectly, and swapping the operands would
         // cause it to handle NaNs incorrectly.
         if (!DAG.getTarget().Options.UnsafeFPMath &&
             !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) {
           if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
             break;
           std::swap(LHS, RHS);
         }
         Opcode = X86ISD::FMIN;
         break;
       case ISD::SETUGT:
         // Converting this to a min would handle NaNs incorrectly.
         if (!DAG.getTarget().Options.UnsafeFPMath &&
             (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)))
           break;
         Opcode = X86ISD::FMIN;
         break;
       case ISD::SETUGE:
         // Converting this to a min would handle both negative zeros and NaNs
         // incorrectly, but we can swap the operands to fix both.
         std::swap(LHS, RHS);
       case ISD::SETOGT:
       case ISD::SETGT:
       case ISD::SETGE:
         Opcode = X86ISD::FMIN;
         break;
 
       case ISD::SETULT:
         // Converting this to a max would handle NaNs incorrectly.
         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
           break;
         Opcode = X86ISD::FMAX;
         break;
       case ISD::SETOLE:
         // Converting this to a max would handle comparisons between positive
         // and negative zero incorrectly, and swapping the operands would
         // cause it to handle NaNs incorrectly.
         if (!DAG.getTarget().Options.UnsafeFPMath &&
             !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) {
           if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
             break;
           std::swap(LHS, RHS);
         }
         Opcode = X86ISD::FMAX;
         break;
       case ISD::SETULE:
         // Converting this to a max would handle both negative zeros and NaNs
         // incorrectly, but we can swap the operands to fix both.
         std::swap(LHS, RHS);
       case ISD::SETOLT:
       case ISD::SETLT:
       case ISD::SETLE:
         Opcode = X86ISD::FMAX;
         break;
       }
     }
 
     if (Opcode)
       return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
   }
 
   EVT CondVT = Cond.getValueType();
   if (Subtarget->hasAVX512() && VT.isVector() && CondVT.isVector() &&
       CondVT.getVectorElementType() == MVT::i1) {
     // v16i8 (select v16i1, v16i8, v16i8) does not have a proper
     // lowering on AVX-512. In this case we convert it to
     // v16i8 (select v16i8, v16i8, v16i8) and use AVX instruction.
     // The same situation for all 128 and 256-bit vectors of i8 and i16
     EVT OpVT = LHS.getValueType();
     if ((OpVT.is128BitVector() || OpVT.is256BitVector()) &&
         (OpVT.getVectorElementType() == MVT::i8 ||
          OpVT.getVectorElementType() == MVT::i16)) {
       Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, OpVT, Cond);
       DCI.AddToWorklist(Cond.getNode());
       return DAG.getNode(N->getOpcode(), DL, OpVT, Cond, LHS, RHS);
     }
   }
   // If this is a select between two integer constants, try to do some
   // optimizations.
   if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(LHS)) {
     if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(RHS))
       // Don't do this for crazy integer types.
       if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) {
         // If this is efficiently invertible, canonicalize the LHSC/RHSC values
         // so that TrueC (the true value) is larger than FalseC.
         bool NeedsCondInvert = false;
 
         if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue()) &&
             // Efficiently invertible.
             (Cond.getOpcode() == ISD::SETCC ||  // setcc -> invertible.
              (Cond.getOpcode() == ISD::XOR &&   // xor(X, C) -> invertible.
               isa<ConstantSDNode>(Cond.getOperand(1))))) {
           NeedsCondInvert = true;
           std::swap(TrueC, FalseC);
         }
 
         // Optimize C ? 8 : 0 -> zext(C) << 3.  Likewise for any pow2/0.
         if (FalseC->getAPIntValue() == 0 &&
             TrueC->getAPIntValue().isPowerOf2()) {
           if (NeedsCondInvert) // Invert the condition if needed.
             Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
                                DAG.getConstant(1, Cond.getValueType()));
 
           // Zero extend the condition if needed.
           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond);
 
           unsigned ShAmt = TrueC->getAPIntValue().logBase2();
           return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond,
                              DAG.getConstant(ShAmt, MVT::i8));
         }
 
         // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.
         if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
           if (NeedsCondInvert) // Invert the condition if needed.
             Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
                                DAG.getConstant(1, Cond.getValueType()));
 
           // Zero extend the condition if needed.
           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
                              FalseC->getValueType(0), Cond);
           return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
                              SDValue(FalseC, 0));
         }
 
         // Optimize cases that will turn into an LEA instruction.  This requires
         // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
         if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
           uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
           if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
 
           bool isFastMultiplier = false;
           if (Diff < 10) {
             switch ((unsigned char)Diff) {
               default: break;
               case 1:  // result = add base, cond
               case 2:  // result = lea base(    , cond*2)
               case 3:  // result = lea base(cond, cond*2)
               case 4:  // result = lea base(    , cond*4)
               case 5:  // result = lea base(cond, cond*4)
               case 8:  // result = lea base(    , cond*8)
               case 9:  // result = lea base(cond, cond*8)
                 isFastMultiplier = true;
                 break;
             }
           }
 
           if (isFastMultiplier) {
             APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
             if (NeedsCondInvert) // Invert the condition if needed.
               Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
                                  DAG.getConstant(1, Cond.getValueType()));
 
             // Zero extend the condition if needed.
             Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
                                Cond);
             // Scale the condition by the difference.
             if (Diff != 1)
               Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
                                  DAG.getConstant(Diff, Cond.getValueType()));
 
             // Add the base if non-zero.
             if (FalseC->getAPIntValue() != 0)
               Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
                                  SDValue(FalseC, 0));
             return Cond;
           }
         }
       }
   }
 
   // Canonicalize max and min:
   // (x > y) ? x : y -> (x >= y) ? x : y
   // (x < y) ? x : y -> (x <= y) ? x : y
   // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates
   // the need for an extra compare
   // against zero. e.g.
   // (x - y) > 0 : (x - y) ? 0 -> (x - y) >= 0 : (x - y) ? 0
   // subl   %esi, %edi
   // testl  %edi, %edi
   // movl   $0, %eax
   // cmovgl %edi, %eax
   // =>
   // xorl   %eax, %eax
   // subl   %esi, $edi
   // cmovsl %eax, %edi
   if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC &&
       DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
       DAG.isEqualTo(RHS, Cond.getOperand(1))) {
     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
     switch (CC) {
     default: break;
     case ISD::SETLT:
     case ISD::SETGT: {
       ISD::CondCode NewCC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGE;
       Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(),
                           Cond.getOperand(0), Cond.getOperand(1), NewCC);
       return DAG.getNode(ISD::SELECT, DL, VT, Cond, LHS, RHS);
     }
     }
   }
 
   // Early exit check
   if (!TLI.isTypeLegal(VT))
     return SDValue();
 
   // Match VSELECTs into subs with unsigned saturation.
   if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
       // psubus is available in SSE2 and AVX2 for i8 and i16 vectors.
       ((Subtarget->hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) ||
        (Subtarget->hasAVX2() && (VT == MVT::v32i8 || VT == MVT::v16i16)))) {
     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
 
     // Check if one of the arms of the VSELECT is a zero vector. If it's on the
     // left side invert the predicate to simplify logic below.
     SDValue Other;
     if (ISD::isBuildVectorAllZeros(LHS.getNode())) {
       Other = RHS;
       CC = ISD::getSetCCInverse(CC, true);
     } else if (ISD::isBuildVectorAllZeros(RHS.getNode())) {
       Other = LHS;
     }
 
     if (Other.getNode() && Other->getNumOperands() == 2 &&
         DAG.isEqualTo(Other->getOperand(0), Cond.getOperand(0))) {
       SDValue OpLHS = Other->getOperand(0), OpRHS = Other->getOperand(1);
       SDValue CondRHS = Cond->getOperand(1);
 
       // Look for a general sub with unsigned saturation first.
       // x >= y ? x-y : 0 --> subus x, y
       // x >  y ? x-y : 0 --> subus x, y
       if ((CC == ISD::SETUGE || CC == ISD::SETUGT) &&
           Other->getOpcode() == ISD::SUB && DAG.isEqualTo(OpRHS, CondRHS))
         return DAG.getNode(X86ISD::SUBUS, DL, VT, OpLHS, OpRHS);
 
       if (auto *OpRHSBV = dyn_cast<BuildVectorSDNode>(OpRHS))
         if (auto *OpRHSConst = OpRHSBV->getConstantSplatNode()) {
           if (auto *CondRHSBV = dyn_cast<BuildVectorSDNode>(CondRHS))
             if (auto *CondRHSConst = CondRHSBV->getConstantSplatNode())
               // If the RHS is a constant we have to reverse the const
               // canonicalization.
               // x > C-1 ? x+-C : 0 --> subus x, C
               if (CC == ISD::SETUGT && Other->getOpcode() == ISD::ADD &&
                   CondRHSConst->getAPIntValue() ==
                       (-OpRHSConst->getAPIntValue() - 1))
                 return DAG.getNode(
                     X86ISD::SUBUS, DL, VT, OpLHS,
                     DAG.getConstant(-OpRHSConst->getAPIntValue(), VT));
 
           // Another special case: If C was a sign bit, the sub has been
           // canonicalized into a xor.
           // FIXME: Would it be better to use computeKnownBits to determine
           //        whether it's safe to decanonicalize the xor?
           // x s< 0 ? x^C : 0 --> subus x, C
           if (CC == ISD::SETLT && Other->getOpcode() == ISD::XOR &&
               ISD::isBuildVectorAllZeros(CondRHS.getNode()) &&
               OpRHSConst->getAPIntValue().isSignBit())
             // Note that we have to rebuild the RHS constant here to ensure we
             // don't rely on particular values of undef lanes.
             return DAG.getNode(
                 X86ISD::SUBUS, DL, VT, OpLHS,
                 DAG.getConstant(OpRHSConst->getAPIntValue(), VT));
         }
     }
   }
 
   // Try to match a min/max vector operation.
   if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC) {
     std::pair<unsigned, bool> ret = matchIntegerMINMAX(Cond, VT, LHS, RHS, DAG, Subtarget);
     unsigned Opc = ret.first;
     bool NeedSplit = ret.second;
 
     if (Opc && NeedSplit) {
       unsigned NumElems = VT.getVectorNumElements();
       // Extract the LHS vectors
       SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, DL);
       SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, DL);
 
       // Extract the RHS vectors
       SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, DL);
       SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, DL);
 
       // Create min/max for each subvector
       LHS = DAG.getNode(Opc, DL, LHS1.getValueType(), LHS1, RHS1);
       RHS = DAG.getNode(Opc, DL, LHS2.getValueType(), LHS2, RHS2);
 
       // Merge the result
       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LHS, RHS);
     } else if (Opc)
       return DAG.getNode(Opc, DL, VT, LHS, RHS);
   }
 
   // Simplify vector selection if the selector will be produced by CMPP*/PCMP*.
   if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
       // Check if SETCC has already been promoted
       TLI.getSetCCResultType(*DAG.getContext(), VT) == CondVT &&
       // Check that condition value type matches vselect operand type
       CondVT == VT) { 
 
     assert(Cond.getValueType().isVector() &&
            "vector select expects a vector selector!");
 
     bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
     bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
 
     if (!TValIsAllOnes && !FValIsAllZeros) {
       // Try invert the condition if true value is not all 1s and false value
       // is not all 0s.
       bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
       bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
 
       if (TValIsAllZeros || FValIsAllOnes) {
         SDValue CC = Cond.getOperand(2);
         ISD::CondCode NewCC =
           ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
                                Cond.getOperand(0).getValueType().isInteger());
         Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1), NewCC);
         std::swap(LHS, RHS);
         TValIsAllOnes = FValIsAllOnes;
         FValIsAllZeros = TValIsAllZeros;
       }
     }
 
     if (TValIsAllOnes || FValIsAllZeros) {
       SDValue Ret;
 
       if (TValIsAllOnes && FValIsAllZeros)
         Ret = Cond;
       else if (TValIsAllOnes)
         Ret = DAG.getNode(ISD::OR, DL, CondVT, Cond,
                           DAG.getNode(ISD::BITCAST, DL, CondVT, RHS));
       else if (FValIsAllZeros)
         Ret = DAG.getNode(ISD::AND, DL, CondVT, Cond,
                           DAG.getNode(ISD::BITCAST, DL, CondVT, LHS));
 
       return DAG.getNode(ISD::BITCAST, DL, VT, Ret);
     }
   }
 
   // Try to fold this VSELECT into a MOVSS/MOVSD
   if (N->getOpcode() == ISD::VSELECT &&
       Cond.getOpcode() == ISD::BUILD_VECTOR && !DCI.isBeforeLegalize()) {
     if (VT == MVT::v4i32 || VT == MVT::v4f32 ||
         (Subtarget->hasSSE2() && (VT == MVT::v2i64 || VT == MVT::v2f64))) {
       bool CanFold = false;
       unsigned NumElems = Cond.getNumOperands();
       SDValue A = LHS;
       SDValue B = RHS;
       
       if (isZero(Cond.getOperand(0))) {
         CanFold = true;
 
         // fold (vselect <0,-1,-1,-1>, A, B) -> (movss A, B)
         // fold (vselect <0,-1> -> (movsd A, B)
         for (unsigned i = 1, e = NumElems; i != e && CanFold; ++i)
           CanFold = isAllOnes(Cond.getOperand(i));
       } else if (isAllOnes(Cond.getOperand(0))) {
         CanFold = true;
         std::swap(A, B);
 
         // fold (vselect <-1,0,0,0>, A, B) -> (movss B, A)
         // fold (vselect <-1,0> -> (movsd B, A)
         for (unsigned i = 1, e = NumElems; i != e && CanFold; ++i)
           CanFold = isZero(Cond.getOperand(i));
       }
 
       if (CanFold) {
         if (VT == MVT::v4i32 || VT == MVT::v4f32)
           return getTargetShuffleNode(X86ISD::MOVSS, DL, VT, A, B, DAG);
         return getTargetShuffleNode(X86ISD::MOVSD, DL, VT, A, B, DAG);
       }
 
       if (Subtarget->hasSSE2() && (VT == MVT::v4i32 || VT == MVT::v4f32)) {
         // fold (v4i32: vselect <0,0,-1,-1>, A, B) ->
         //      (v4i32 (bitcast (movsd (v2i64 (bitcast A)),
         //                             (v2i64 (bitcast B)))))
         //
         // fold (v4f32: vselect <0,0,-1,-1>, A, B) ->
         //      (v4f32 (bitcast (movsd (v2f64 (bitcast A)),
         //                             (v2f64 (bitcast B)))))
         //
         // fold (v4i32: vselect <-1,-1,0,0>, A, B) ->
         //      (v4i32 (bitcast (movsd (v2i64 (bitcast B)),
         //                             (v2i64 (bitcast A)))))
         //
         // fold (v4f32: vselect <-1,-1,0,0>, A, B) ->
         //      (v4f32 (bitcast (movsd (v2f64 (bitcast B)),
         //                             (v2f64 (bitcast A)))))
 
         CanFold = (isZero(Cond.getOperand(0)) &&
                    isZero(Cond.getOperand(1)) &&
                    isAllOnes(Cond.getOperand(2)) &&
                    isAllOnes(Cond.getOperand(3)));
 
         if (!CanFold && isAllOnes(Cond.getOperand(0)) &&
             isAllOnes(Cond.getOperand(1)) &&
             isZero(Cond.getOperand(2)) &&
             isZero(Cond.getOperand(3))) {
           CanFold = true;
           std::swap(LHS, RHS);
         }
 
         if (CanFold) {
           EVT NVT = (VT == MVT::v4i32) ? MVT::v2i64 : MVT::v2f64;
           SDValue NewA = DAG.getNode(ISD::BITCAST, DL, NVT, LHS);
           SDValue NewB = DAG.getNode(ISD::BITCAST, DL, NVT, RHS);
           SDValue Select = getTargetShuffleNode(X86ISD::MOVSD, DL, NVT, NewA,
                                                 NewB, DAG);
           return DAG.getNode(ISD::BITCAST, DL, VT, Select);
         }
       }
     }
   }
 
   // If we know that this node is legal then we know that it is going to be
   // matched by one of the SSE/AVX BLEND instructions. These instructions only
   // depend on the highest bit in each word. Try to use SimplifyDemandedBits
   // to simplify previous instructions.
   if (N->getOpcode() == ISD::VSELECT && DCI.isBeforeLegalizeOps() &&
       !DCI.isBeforeLegalize() &&
       // We explicitly check against v8i16 and v16i16 because, although
       // they're marked as Custom, they might only be legal when Cond is a
       // build_vector of constants. This will be taken care in a later
       // condition.
       (TLI.isOperationLegalOrCustom(ISD::VSELECT, VT) && VT != MVT::v16i16 &&
        VT != MVT::v8i16)) {
     unsigned BitWidth = Cond.getValueType().getScalarType().getSizeInBits();
 
     // Don't optimize vector selects that map to mask-registers.
     if (BitWidth == 1)
       return SDValue();
 
     // Check all uses of that condition operand to check whether it will be
     // consumed by non-BLEND instructions, which may depend on all bits are set
     // properly.
     for (SDNode::use_iterator I = Cond->use_begin(),
                               E = Cond->use_end(); I != E; ++I)
       if (I->getOpcode() != ISD::VSELECT)
         // TODO: Add other opcodes eventually lowered into BLEND.
         return SDValue();
 
     assert(BitWidth >= 8 && BitWidth <= 64 && "Invalid mask size");
     APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 1);
 
     APInt KnownZero, KnownOne;
     TargetLowering::TargetLoweringOpt TLO(DAG, DCI.isBeforeLegalize(),
                                           DCI.isBeforeLegalizeOps());
     if (TLO.ShrinkDemandedConstant(Cond, DemandedMask) ||
         TLI.SimplifyDemandedBits(Cond, DemandedMask, KnownZero, KnownOne, TLO))
       DCI.CommitTargetLoweringOpt(TLO);
   }
 
   // We should generate an X86ISD::BLENDI from a vselect if its argument
   // is a sign_extend_inreg of an any_extend of a BUILD_VECTOR of
   // constants. This specific pattern gets generated when we split a
   // selector for a 512 bit vector in a machine without AVX512 (but with
   // 256-bit vectors), during legalization:
   //
   // (vselect (sign_extend (any_extend (BUILD_VECTOR)) i1) LHS RHS)
   //
   // Iff we find this pattern and the build_vectors are built from
   // constants, we translate the vselect into a shuffle_vector that we
   // know will be matched by LowerVECTOR_SHUFFLEtoBlend.
   if (N->getOpcode() == ISD::VSELECT && !DCI.isBeforeLegalize()) {
     SDValue Shuffle = TransformVSELECTtoBlendVECTOR_SHUFFLE(N, DAG, Subtarget);
     if (Shuffle.getNode())
       return Shuffle;
   }
 
   return SDValue();
 }

static SDValue PerformSETCCCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 21782 of file X86ISelLowering.cpp.

                                                                   {
   SDLoc DL(N);
   X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
   SDValue EFLAGS = N->getOperand(1);
 
   if (CC == X86::COND_A) {
     // Try to convert COND_A into COND_B in an attempt to facilitate
     // materializing "setb reg".
     //
     // Do not flip "e > c", where "c" is a constant, because Cmp instruction
     // cannot take an immediate as its first operand.
     //
     if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
         EFLAGS.getValueType().isInteger() &&
         !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
       SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
                                    EFLAGS.getNode()->getVTList(),
                                    EFLAGS.getOperand(1), EFLAGS.getOperand(0));
       SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
       return MaterializeSETB(DL, NewEFLAGS, DAG, N->getSimpleValueType(0));
     }
   }
 
   // Materialize "setb reg" as "sbb reg,reg", since it can be extended without
   // a zext and produces an all-ones bit which is more useful than 0/1 in some
   // cases.
   if (CC == X86::COND_B)
     return MaterializeSETB(DL, EFLAGS, DAG, N->getSimpleValueType(0));
 
   SDValue Flags;
 
   Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
   if (Flags.getNode()) {
     SDValue Cond = DAG.getConstant(CC, MVT::i8);
     return DAG.getNode(X86ISD::SETCC, DL, N->getVTList(), Cond, Flags);
   }
 
   return SDValue();
 }

static SDValue PerformSExtCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 21576 of file X86ISelLowering.cpp.

                                                                  {
   if (!DCI.isBeforeLegalizeOps())
     return SDValue();
 
   if (!Subtarget->hasFp256())
     return SDValue();
 
   EVT VT = N->getValueType(0);
   if (VT.isVector() && VT.getSizeInBits() == 256) {
     SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget);
     if (R.getNode())
       return R;
   }
 
   return SDValue();
 }

static SDValue PerformShiftCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

PerformShiftCombine - Combine shifts.

Definition at line 20359 of file X86ISelLowering.cpp.

                                                                   {
   if (N->getOpcode() == ISD::SHL) {
     SDValue V = PerformSHLCombine(N, DAG);
     if (V.getNode()) return V;
   }
 
   if (N->getOpcode() != ISD::SRA) {
     // Try to fold this logical shift into a zero vector.
     SDValue V = performShiftToAllZeros(N, DAG, Subtarget);
     if (V.getNode()) return V;
   }
 
   return SDValue();
 }

static SDValue performShiftToAllZeros	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits.

Definition at line 20331 of file X86ISelLowering.cpp.

                                                                      {
   EVT VT = N->getValueType(0);
 
   if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 &&
       (!Subtarget->hasInt256() ||
        (VT != MVT::v4i64 && VT != MVT::v8i32 && VT != MVT::v16i16)))
     return SDValue();
 
   SDValue Amt = N->getOperand(1);
   SDLoc DL(N);
   if (auto *AmtBV = dyn_cast<BuildVectorSDNode>(Amt))
     if (auto *AmtSplat = AmtBV->getConstantSplatNode()) {
       APInt ShiftAmt = AmtSplat->getAPIntValue();
       unsigned MaxAmount = VT.getVectorElementType().getSizeInBits();
 
       // SSE2/AVX2 logical shifts always return a vector of 0s
       // if the shift amount is bigger than or equal to
       // the element size. The constant shift amount will be
       // encoded as a 8-bit immediate.
       if (ShiftAmt.trunc(8).uge(MaxAmount))
         return getZeroVector(VT, Subtarget, DAG, DL);
     }
 
   return SDValue();
 }

static SDValue PerformSHLCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 20286 of file X86ISelLowering.cpp.

                                                                {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   EVT VT = N0.getValueType();
 
   // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
   // since the result of setcc_c is all zero's or all ones.
   if (VT.isInteger() && !VT.isVector() &&
       N1C && N0.getOpcode() == ISD::AND &&
       N0.getOperand(1).getOpcode() == ISD::Constant) {
     SDValue N00 = N0.getOperand(0);
     if (N00.getOpcode() == X86ISD::SETCC_CARRY ||
         ((N00.getOpcode() == ISD::ANY_EXTEND ||
           N00.getOpcode() == ISD::ZERO_EXTEND) &&
          N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY)) {
       APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
       APInt ShAmt = N1C->getAPIntValue();
       Mask = Mask.shl(ShAmt);
       if (Mask != 0)
         return DAG.getNode(ISD::AND, SDLoc(N), VT,
                            N00, DAG.getConstant(Mask, VT));
     }
   }
 
   // Hardware support for vector shifts is sparse which makes us scalarize the
   // vector operations in many cases. Also, on sandybridge ADD is faster than
   // shl.
   // (shl V, 1) -> add V,V
   if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
     if (auto *N1SplatC = N1BV->getConstantSplatNode()) {
       assert(N0.getValueType().isVector() && "Invalid vector shift type");
       // We shift all of the values by one. In many cases we do not have
       // hardware support for this operation. This is better expressed as an ADD
       // of two values.
       if (N1SplatC->getZExtValue() == 1)
         return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0);
     }
 
   return SDValue();
 }

static SDValue PerformShuffleCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

PerformShuffleCombine - Performs several different shuffle combines.

Definition at line 18792 of file X86ISelLowering.cpp.

                                                                     {
   SDLoc dl(N);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
 
   // Canonicalize shuffles that perform 'addsub' on packed float vectors
   // according to the rule:
   //  (shuffle (FADD A, B), (FSUB A, B), Mask) ->
   //  (shuffle (FSUB A, -B), (FADD A, -B), Mask)
   //
   // Where 'Mask' is:
   //  <0,5,2,7>             -- for v4f32 and v4f64 shuffles;
   //  <0,3>                 -- for v2f64 shuffles;
   //  <0,9,2,11,4,13,6,15>  -- for v8f32 shuffles.
   //
   // This helps pattern-matching more SSE3/AVX ADDSUB instructions
   // during ISel stage.
   if (N->getOpcode() == ISD::VECTOR_SHUFFLE &&
       ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
        (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
       N0->getOpcode() == ISD::FADD && N1->getOpcode() == ISD::FSUB &&
       // Operands to the FADD and FSUB must be the same.
       ((N0->getOperand(0) == N1->getOperand(0) &&
         N0->getOperand(1) == N1->getOperand(1)) ||
        // FADD is commutable. See if by commuting the operands of the FADD
        // we would still be able to match the operands of the FSUB dag node.
        (N0->getOperand(1) == N1->getOperand(0) &&
         N0->getOperand(0) == N1->getOperand(1))) &&
       N0->getOperand(0)->getOpcode() != ISD::UNDEF &&
       N0->getOperand(1)->getOpcode() != ISD::UNDEF) {
     
     ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
     unsigned NumElts = VT.getVectorNumElements();
     ArrayRef<int> Mask = SV->getMask();
     bool CanFold = true;
 
     for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i)
       CanFold = Mask[i] == (int)((i & 1) ? i + NumElts : i);
 
     if (CanFold) {
       SDValue Op0 = N1->getOperand(0);
       SDValue Op1 = DAG.getNode(ISD::FNEG, dl, VT, N1->getOperand(1));
       SDValue Sub = DAG.getNode(ISD::FSUB, dl, VT, Op0, Op1);
       SDValue Add = DAG.getNode(ISD::FADD, dl, VT, Op0, Op1);
       return DAG.getVectorShuffle(VT, dl, Sub, Add, Mask);
     }
   }
 
   // Don't create instructions with illegal types after legalize types has run.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(VT.getVectorElementType()))
     return SDValue();
 
   // Combine 256-bit vector shuffles. This is only profitable when in AVX mode
   if (Subtarget->hasFp256() && VT.is256BitVector() &&
       N->getOpcode() == ISD::VECTOR_SHUFFLE)
     return PerformShuffleCombine256(N, DAG, DCI, Subtarget);
 
   // During Type Legalization, when promoting illegal vector types,
   // the backend might introduce new shuffle dag nodes and bitcasts.
   //
   // This code performs the following transformation:
   // fold: (shuffle (bitcast (BINOP A, B)), Undef, <Mask>) ->
   //       (shuffle (BINOP (bitcast A), (bitcast B)), Undef, <Mask>)
   //
   // We do this only if both the bitcast and the BINOP dag nodes have
   // one use. Also, perform this transformation only if the new binary
   // operation is legal. This is to avoid introducing dag nodes that
   // potentially need to be further expanded (or custom lowered) into a
   // less optimal sequence of dag nodes.
   if (!DCI.isBeforeLegalize() && DCI.isBeforeLegalizeOps() &&
       N1.getOpcode() == ISD::UNDEF && N0.hasOneUse() &&
       N0.getOpcode() == ISD::BITCAST) {
     SDValue BC0 = N0.getOperand(0);
     EVT SVT = BC0.getValueType();
     unsigned Opcode = BC0.getOpcode();
     unsigned NumElts = VT.getVectorNumElements();
     
     if (BC0.hasOneUse() && SVT.isVector() &&
         SVT.getVectorNumElements() * 2 == NumElts &&
         TLI.isOperationLegal(Opcode, VT)) {
       bool CanFold = false;
       switch (Opcode) {
       default : break;
       case ISD::ADD :
       case ISD::FADD :
       case ISD::SUB :
       case ISD::FSUB :
       case ISD::MUL :
       case ISD::FMUL :
         CanFold = true;
       }
 
       unsigned SVTNumElts = SVT.getVectorNumElements();
       ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
       for (unsigned i = 0, e = SVTNumElts; i != e && CanFold; ++i)
         CanFold = SVOp->getMaskElt(i) == (int)(i * 2);
       for (unsigned i = SVTNumElts, e = NumElts; i != e && CanFold; ++i)
         CanFold = SVOp->getMaskElt(i) < 0;
 
       if (CanFold) {
         SDValue BC00 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(0));
         SDValue BC01 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(1));
         SDValue NewBinOp = DAG.getNode(BC0.getOpcode(), dl, VT, BC00, BC01);
         return DAG.getVectorShuffle(VT, dl, NewBinOp, N1, &SVOp->getMask()[0]);
       }
     }
   }
 
   // Only handle 128 wide vector from here on.
   if (!VT.is128BitVector())
     return SDValue();
 
   // Combine a vector_shuffle that is equal to build_vector load1, load2, load3,
   // load4, <0, 1, 2, 3> into a 128-bit load if the load addresses are
   // consecutive, non-overlapping, and in the right order.
   SmallVector<SDValue, 16> Elts;
   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
     Elts.push_back(getShuffleScalarElt(N, i, DAG, 0));
 
   SDValue LD = EltsFromConsecutiveLoads(VT, Elts, dl, DAG, true);
   if (LD.getNode())
     return LD;
 
   if (isTargetShuffle(N->getOpcode())) {
     SDValue Shuffle =
         PerformTargetShuffleCombine(SDValue(N, 0), DAG, DCI, Subtarget);
     if (Shuffle.getNode())
       return Shuffle;
   }
 
   return SDValue();
 }

static SDValue PerformShuffleCombine256	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors.

Definition at line 18370 of file X86ISelLowering.cpp.

                                                                        {
   SDLoc dl(N);
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   EVT VT = SVOp->getValueType(0);
   unsigned NumElems = VT.getVectorNumElements();
 
   if (V1.getOpcode() == ISD::CONCAT_VECTORS &&
       V2.getOpcode() == ISD::CONCAT_VECTORS) {
     //
     //                   0,0,0,...
     //                      |
     //    V      UNDEF    BUILD_VECTOR    UNDEF
     //     \      /           \           /
     //  CONCAT_VECTOR         CONCAT_VECTOR
     //         \                  /
     //          \                /
     //          RESULT: V + zero extended
     //
     if (V2.getOperand(0).getOpcode() != ISD::BUILD_VECTOR ||
         V2.getOperand(1).getOpcode() != ISD::UNDEF ||
         V1.getOperand(1).getOpcode() != ISD::UNDEF)
       return SDValue();
 
     if (!ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()))
       return SDValue();
 
     // To match the shuffle mask, the first half of the mask should
     // be exactly the first vector, and all the rest a splat with the
     // first element of the second one.
     for (unsigned i = 0; i != NumElems/2; ++i)
       if (!isUndefOrEqual(SVOp->getMaskElt(i), i) ||
           !isUndefOrEqual(SVOp->getMaskElt(i+NumElems/2), NumElems))
         return SDValue();
 
     // If V1 is coming from a vector load then just fold to a VZEXT_LOAD.
     if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(V1.getOperand(0))) {
       if (Ld->hasNUsesOfValue(1, 0)) {
         SDVTList Tys = DAG.getVTList(MVT::v4i64, MVT::Other);
         SDValue Ops[] = { Ld->getChain(), Ld->getBasePtr() };
         SDValue ResNode =
           DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
                                   Ld->getMemoryVT(),
                                   Ld->getPointerInfo(),
                                   Ld->getAlignment(),
                                   false/*isVolatile*/, true/*ReadMem*/,
                                   false/*WriteMem*/);
 
         // Make sure the newly-created LOAD is in the same position as Ld in
         // terms of dependency. We create a TokenFactor for Ld and ResNode,
         // and update uses of Ld's output chain to use the TokenFactor.
         if (Ld->hasAnyUseOfValue(1)) {
           SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                              SDValue(Ld, 1), SDValue(ResNode.getNode(), 1));
           DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), NewChain);
           DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(Ld, 1),
                                  SDValue(ResNode.getNode(), 1));
         }
 
         return DAG.getNode(ISD::BITCAST, dl, VT, ResNode);
       }
     }
 
     // Emit a zeroed vector and insert the desired subvector on its
     // first half.
     SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
     SDValue InsV = Insert128BitVector(Zeros, V1.getOperand(0), 0, DAG, dl);
     return DCI.CombineTo(N, InsV);
   }
 
   //===--------------------------------------------------------------------===//
   // Combine some shuffles into subvector extracts and inserts:
   //
 
   // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
   if (isShuffleHigh128VectorInsertLow(SVOp)) {
     SDValue V = Extract128BitVector(V1, NumElems/2, DAG, dl);
     SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, 0, DAG, dl);
     return DCI.CombineTo(N, InsV);
   }
 
   // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
   if (isShuffleLow128VectorInsertHigh(SVOp)) {
     SDValue V = Extract128BitVector(V1, 0, DAG, dl);
     SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, NumElems/2, DAG, dl);
     return DCI.CombineTo(N, InsV);
   }
 
   return SDValue();
 }

static SDValue PerformSIGN_EXTEND_INREGCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 21541 of file X86ISelLowering.cpp.

                                                                               {
   EVT VT = N->getValueType(0);
   if (!VT.isVector())
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
   SDLoc dl(N);
 
   // The SIGN_EXTEND_INREG to v4i64 is expensive operation on the
   // both SSE and AVX2 since there is no sign-extended shift right
   // operation on a vector with 64-bit elements.
   //(sext_in_reg (v4i64 anyext (v4i32 x )), ExtraVT) ->
   // (v4i64 sext (v4i32 sext_in_reg (v4i32 x , ExtraVT)))
   if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||
       N0.getOpcode() == ISD::SIGN_EXTEND)) {
     SDValue N00 = N0.getOperand(0);
 
     // EXTLOAD has a better solution on AVX2,
     // it may be replaced with X86ISD::VSEXT node.
     if (N00.getOpcode() == ISD::LOAD && Subtarget->hasInt256())
       if (!ISD::isNormalLoad(N00.getNode()))
         return SDValue();
 
     if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) {
         SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32,
                                   N00, N1);
       return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);
     }
   }
   return SDValue();
 }

static SDValue PerformSINT_TO_FPCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86TargetLowering *	XTLI
	)

static

Definition at line 21893 of file X86ISelLowering.cpp.

                                                                        {
   // First try to optimize away the conversion entirely when it's
   // conditionally from a constant. Vectors only.
   SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG);
   if (Res != SDValue())
     return Res;
 
   // Now move on to more general possibilities.
   SDValue Op0 = N->getOperand(0);
   EVT InVT = Op0->getValueType(0);
 
   // SINT_TO_FP(v4i8) -> SINT_TO_FP(SEXT(v4i8 to v4i32))
   if (InVT == MVT::v8i8 || InVT == MVT::v4i8) {
     SDLoc dl(N);
     MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32;
     SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
     return DAG.getNode(ISD::SINT_TO_FP, dl, N->getValueType(0), P);
   }
 
   // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have
   // a 32-bit target where SSE doesn't support i64->FP operations.
   if (Op0.getOpcode() == ISD::LOAD) {
     LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode());
     EVT VT = Ld->getValueType(0);
     if (!Ld->isVolatile() && !N->getValueType(0).isVector() &&
         ISD::isNON_EXTLoad(Op0.getNode()) && Op0.hasOneUse() &&
         !XTLI->getSubtarget()->is64Bit() &&
         VT == MVT::i64) {
       SDValue FILDChain = XTLI->BuildFILD(SDValue(N, 0), Ld->getValueType(0),
                                           Ld->getChain(), Op0, DAG);
       DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), FILDChain.getValue(1));
       return FILDChain;
     }
   }
   return SDValue();
 }

static SDValue PerformSTORECombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

PerformSTORECombine - Do target-specific dag combines on STORE nodes.

Definition at line 21061 of file X86ISelLowering.cpp.

                                                                   {
   StoreSDNode *St = cast<StoreSDNode>(N);
   EVT VT = St->getValue().getValueType();
   EVT StVT = St->getMemoryVT();
   SDLoc dl(St);
   SDValue StoredVal = St->getOperand(1);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // If we are saving a concatenation of two XMM registers, perform two stores.
   // On Sandy Bridge, 256-bit memory operations are executed by two
   // 128-bit ports. However, on Haswell it is better to issue a single 256-bit
   // memory  operation.
   unsigned Alignment = St->getAlignment();
   bool IsAligned = Alignment == 0 || Alignment >= VT.getSizeInBits()/8;
   if (VT.is256BitVector() && !Subtarget->hasInt256() &&
       StVT == VT && !IsAligned) {
     unsigned NumElems = VT.getVectorNumElements();
     if (NumElems < 2)
       return SDValue();
 
     SDValue Value0 = Extract128BitVector(StoredVal, 0, DAG, dl);
     SDValue Value1 = Extract128BitVector(StoredVal, NumElems/2, DAG, dl);
 
     SDValue Stride = DAG.getConstant(16, TLI.getPointerTy());
     SDValue Ptr0 = St->getBasePtr();
     SDValue Ptr1 = DAG.getNode(ISD::ADD, dl, Ptr0.getValueType(), Ptr0, Stride);
 
     SDValue Ch0 = DAG.getStore(St->getChain(), dl, Value0, Ptr0,
                                 St->getPointerInfo(), St->isVolatile(),
                                 St->isNonTemporal(), Alignment);
     SDValue Ch1 = DAG.getStore(St->getChain(), dl, Value1, Ptr1,
                                 St->getPointerInfo(), St->isVolatile(),
                                 St->isNonTemporal(),
                                 std::min(16U, Alignment));
     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
   }
 
   // Optimize trunc store (of multiple scalars) to shuffle and store.
   // First, pack all of the elements in one place. Next, store to memory
   // in fewer chunks.
   if (St->isTruncatingStore() && VT.isVector()) {
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     unsigned NumElems = VT.getVectorNumElements();
     assert(StVT != VT && "Cannot truncate to the same type");
     unsigned FromSz = VT.getVectorElementType().getSizeInBits();
     unsigned ToSz = StVT.getVectorElementType().getSizeInBits();
 
     // From, To sizes and ElemCount must be pow of two
     if (!isPowerOf2_32(NumElems * FromSz * ToSz)) return SDValue();
     // We are going to use the original vector elt for storing.
     // Accumulated smaller vector elements must be a multiple of the store size.
     if (0 != (NumElems * FromSz) % ToSz) return SDValue();
 
     unsigned SizeRatio  = FromSz / ToSz;
 
     assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
 
     // Create a type on which we perform the shuffle
     EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
             StVT.getScalarType(), NumElems*SizeRatio);
 
     assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
 
     SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, St->getValue());
     SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
     for (unsigned i = 0; i != NumElems; ++i)
       ShuffleVec[i] = i * SizeRatio;
 
     // Can't shuffle using an illegal type.
     if (!TLI.isTypeLegal(WideVecVT))
       return SDValue();
 
     SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, WideVec,
                                          DAG.getUNDEF(WideVecVT),
                                          &ShuffleVec[0]);
     // At this point all of the data is stored at the bottom of the
     // register. We now need to save it to mem.
 
     // Find the largest store unit
     MVT StoreType = MVT::i8;
     for (unsigned tp = MVT::FIRST_INTEGER_VALUETYPE;
          tp < MVT::LAST_INTEGER_VALUETYPE; ++tp) {
       MVT Tp = (MVT::SimpleValueType)tp;
       if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToSz)
         StoreType = Tp;
     }
 
     // On 32bit systems, we can't save 64bit integers. Try bitcasting to F64.
     if (TLI.isTypeLegal(MVT::f64) && StoreType.getSizeInBits() < 64 &&
         (64 <= NumElems * ToSz))
       StoreType = MVT::f64;
 
     // Bitcast the original vector into a vector of store-size units
     EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
             StoreType, VT.getSizeInBits()/StoreType.getSizeInBits());
     assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
     SDValue ShuffWide = DAG.getNode(ISD::BITCAST, dl, StoreVecVT, Shuff);
     SmallVector<SDValue, 8> Chains;
     SDValue Increment = DAG.getConstant(StoreType.getSizeInBits()/8,
                                         TLI.getPointerTy());
     SDValue Ptr = St->getBasePtr();
 
     // Perform one or more big stores into memory.
     for (unsigned i=0, e=(ToSz*NumElems)/StoreType.getSizeInBits(); i!=e; ++i) {
       SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
                                    StoreType, ShuffWide,
                                    DAG.getIntPtrConstant(i));
       SDValue Ch = DAG.getStore(St->getChain(), dl, SubVec, Ptr,
                                 St->getPointerInfo(), St->isVolatile(),
                                 St->isNonTemporal(), St->getAlignment());
       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
       Chains.push_back(Ch);
     }
 
     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
   }
 
   // Turn load->store of MMX types into GPR load/stores.  This avoids clobbering
   // the FP state in cases where an emms may be missing.
   // A preferable solution to the general problem is to figure out the right
   // places to insert EMMS.  This qualifies as a quick hack.
 
   // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
   if (VT.getSizeInBits() != 64)
     return SDValue();
 
   const Function *F = DAG.getMachineFunction().getFunction();
   bool NoImplicitFloatOps = F->getAttributes().
     hasAttribute(AttributeSet::FunctionIndex, Attribute::NoImplicitFloat);
   bool F64IsLegal = !DAG.getTarget().Options.UseSoftFloat && !NoImplicitFloatOps
                      && Subtarget->hasSSE2();
   if ((VT.isVector() ||
        (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) &&
       isa<LoadSDNode>(St->getValue()) &&
       !cast<LoadSDNode>(St->getValue())->isVolatile() &&
       St->getChain().hasOneUse() && !St->isVolatile()) {
     SDNode* LdVal = St->getValue().getNode();
     LoadSDNode *Ld = nullptr;
     int TokenFactorIndex = -1;
     SmallVector<SDValue, 8> Ops;
     SDNode* ChainVal = St->getChain().getNode();
     // Must be a store of a load.  We currently handle two cases:  the load
     // is a direct child, and it's under an intervening TokenFactor.  It is
     // possible to dig deeper under nested TokenFactors.
     if (ChainVal == LdVal)
       Ld = cast<LoadSDNode>(St->getChain());
     else if (St->getValue().hasOneUse() &&
              ChainVal->getOpcode() == ISD::TokenFactor) {
       for (unsigned i = 0, e = ChainVal->getNumOperands(); i != e; ++i) {
         if (ChainVal->getOperand(i).getNode() == LdVal) {
           TokenFactorIndex = i;
           Ld = cast<LoadSDNode>(St->getValue());
         } else
           Ops.push_back(ChainVal->getOperand(i));
       }
     }
 
     if (!Ld || !ISD::isNormalLoad(Ld))
       return SDValue();
 
     // If this is not the MMX case, i.e. we are just turning i64 load/store
     // into f64 load/store, avoid the transformation if there are multiple
     // uses of the loaded value.
     if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0))
       return SDValue();
 
     SDLoc LdDL(Ld);
     SDLoc StDL(N);
     // If we are a 64-bit capable x86, lower to a single movq load/store pair.
     // Otherwise, if it's legal to use f64 SSE instructions, use f64 load/store
     // pair instead.
     if (Subtarget->is64Bit() || F64IsLegal) {
       EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64;
       SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(), Ld->getBasePtr(),
                                   Ld->getPointerInfo(), Ld->isVolatile(),
                                   Ld->isNonTemporal(), Ld->isInvariant(),
                                   Ld->getAlignment());
       SDValue NewChain = NewLd.getValue(1);
       if (TokenFactorIndex != -1) {
         Ops.push_back(NewChain);
         NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops);
       }
       return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(),
                           St->getPointerInfo(),
                           St->isVolatile(), St->isNonTemporal(),
                           St->getAlignment());
     }
 
     // Otherwise, lower to two pairs of 32-bit loads / stores.
     SDValue LoAddr = Ld->getBasePtr();
     SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr,
                                  DAG.getConstant(4, MVT::i32));
 
     SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr,
                                Ld->getPointerInfo(),
                                Ld->isVolatile(), Ld->isNonTemporal(),
                                Ld->isInvariant(), Ld->getAlignment());
     SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr,
                                Ld->getPointerInfo().getWithOffset(4),
                                Ld->isVolatile(), Ld->isNonTemporal(),
                                Ld->isInvariant(),
                                MinAlign(Ld->getAlignment(), 4));
 
     SDValue NewChain = LoLd.getValue(1);
     if (TokenFactorIndex != -1) {
       Ops.push_back(LoLd);
       Ops.push_back(HiLd);
       NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops);
     }
 
     LoAddr = St->getBasePtr();
     HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr,
                          DAG.getConstant(4, MVT::i32));
 
     SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr,
                                 St->getPointerInfo(),
                                 St->isVolatile(), St->isNonTemporal(),
                                 St->getAlignment());
     SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr,
                                 St->getPointerInfo().getWithOffset(4),
                                 St->isVolatile(),
                                 St->isNonTemporal(),
                                 MinAlign(St->getAlignment(), 4));
     return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt);
   }
   return SDValue();
 }

static SDValue PerformSubCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 22012 of file X86ISelLowering.cpp.

                                                                 {
   SDValue Op0 = N->getOperand(0);
   SDValue Op1 = N->getOperand(1);
 
   // X86 can't encode an immediate LHS of a sub. See if we can push the
   // negation into a preceding instruction.
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op0)) {
     // If the RHS of the sub is a XOR with one use and a constant, invert the
     // immediate. Then add one to the LHS of the sub so we can turn
     // X-Y -> X+~Y+1, saving one register.
     if (Op1->hasOneUse() && Op1.getOpcode() == ISD::XOR &&
         isa<ConstantSDNode>(Op1.getOperand(1))) {
       APInt XorC = cast<ConstantSDNode>(Op1.getOperand(1))->getAPIntValue();
       EVT VT = Op0.getValueType();
       SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,
                                    Op1.getOperand(0),
                                    DAG.getConstant(~XorC, VT));
       return DAG.getNode(ISD::ADD, SDLoc(N), VT, NewXor,
                          DAG.getConstant(C->getAPIntValue()+1, VT));
     }
   }
 
   // Try to synthesize horizontal adds from adds of shuffles.
   EVT VT = N->getValueType(0);
   if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) ||
        (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) &&
       isHorizontalBinOp(Op0, Op1, true))
     return DAG.getNode(X86ISD::HSUB, SDLoc(N), VT, Op0, Op1);
 
   return OptimizeConditionalInDecrement(N, DAG);
 }

static SDValue PerformTargetShuffleCombine	(	SDValue	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Try to combine x86 target specific shuffles.

Definition at line 18687 of file X86ISelLowering.cpp.

                                                                           {
   SDLoc DL(N);
   MVT VT = N.getSimpleValueType();
   SmallVector<int, 4> Mask;
 
   switch (N.getOpcode()) {
   case X86ISD::PSHUFD:
   case X86ISD::PSHUFLW:
   case X86ISD::PSHUFHW:
     Mask = getPSHUFShuffleMask(N);
     assert(Mask.size() == 4);
     break;
   default:
     return SDValue();
   }
 
   // Nuke no-op shuffles that show up after combining.
   if (isNoopShuffleMask(Mask))
     return DCI.CombineTo(N.getNode(), N.getOperand(0), /*AddTo*/ true);
 
   // Look for simplifications involving one or two shuffle instructions.
   SDValue V = N.getOperand(0);
   switch (N.getOpcode()) {
   default:
     break;
   case X86ISD::PSHUFLW:
   case X86ISD::PSHUFHW:
     assert(VT == MVT::v8i16);
     (void)VT;
 
     if (combineRedundantHalfShuffle(N, Mask, DAG, DCI))
       return SDValue(); // We combined away this shuffle, so we're done.
 
     // See if this reduces to a PSHUFD which is no more expensive and can
     // combine with more operations.
     if (Mask[0] % 2 == 0 && Mask[2] % 2 == 0 &&
         areAdjacentMasksSequential(Mask)) {
       int DMask[] = {-1, -1, -1, -1};
       int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
       DMask[DOffset + 0] = DOffset + Mask[0] / 2;
       DMask[DOffset + 1] = DOffset + Mask[2] / 2;
       V = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V);
       DCI.AddToWorklist(V.getNode());
       V = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V,
                       getV4X86ShuffleImm8ForMask(DMask, DAG));
       DCI.AddToWorklist(V.getNode());
       return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V);
     }
 
     // Look for shuffle patterns which can be implemented as a single unpack.
     // FIXME: This doesn't handle the location of the PSHUFD generically, and
     // only works when we have a PSHUFD followed by two half-shuffles.
     if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
         (V.getOpcode() == X86ISD::PSHUFLW ||
          V.getOpcode() == X86ISD::PSHUFHW) &&
         V.getOpcode() != N.getOpcode() &&
         V.hasOneUse()) {
       SDValue D = V.getOperand(0);
       while (D.getOpcode() == ISD::BITCAST && D.hasOneUse())
         D = D.getOperand(0);
       if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
         SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
         SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
         int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
         int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
         int WordMask[8];
         for (int i = 0; i < 4; ++i) {
           WordMask[i + NOffset] = Mask[i] + NOffset;
           WordMask[i + VOffset] = VMask[i] + VOffset;
         }
         // Map the word mask through the DWord mask.
         int MappedMask[8];
         for (int i = 0; i < 8; ++i)
           MappedMask[i] = 2 * DMask[WordMask[i] / 2] + WordMask[i] % 2;
         const int UnpackLoMask[] = {0, 0, 1, 1, 2, 2, 3, 3};
         const int UnpackHiMask[] = {4, 4, 5, 5, 6, 6, 7, 7};
         if (std::equal(std::begin(MappedMask), std::end(MappedMask),
                        std::begin(UnpackLoMask)) ||
             std::equal(std::begin(MappedMask), std::end(MappedMask),
                        std::begin(UnpackHiMask))) {
           // We can replace all three shuffles with an unpack.
           V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, D.getOperand(0));
           DCI.AddToWorklist(V.getNode());
           return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
                                                 : X86ISD::UNPCKH,
                              DL, MVT::v8i16, V, V);
         }
       }
     }
 
     break;
 
   case X86ISD::PSHUFD:
     if (combineRedundantDWordShuffle(N, Mask, DAG, DCI))
       return SDValue(); // We combined away this shuffle.
 
     break;
   }
 
   return SDValue();
 }

static SDValue PerformTruncateCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

PerformTruncateCombine - Converts truncate operation to a sequence of vector shuffle operations. It is possible when we truncate 256-bit vector to 128-bit vector

Definition at line 18932 of file X86ISelLowering.cpp.

                                                                       {
   return SDValue();
 }

static SDValue performVectorCompareAndMaskUnaryOpCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 21847 of file X86ISelLowering.cpp.

                                                                             {
   // Take advantage of vector comparisons producing 0 or -1 in each lane to
   // optimize away operation when it's from a constant.
   //
   // The general transformation is:
   //    UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
   //       AND(VECTOR_CMP(x,y), constant2)
   //    constant2 = UNARYOP(constant)
 
   // Early exit if this isn't a vector operation, the operand of the
   // unary operation isn't a bitwise AND, or if the sizes of the operations
   // aren't the same.
   EVT VT = N->getValueType(0);
   if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND ||
       N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC ||
       VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits())
     return SDValue();
 
   // Now check that the other operand of the AND is a constant splat. We could
   // make the transformation for non-constant splats as well, but it's unclear
   // that would be a benefit as it would not eliminate any operations, just
   // perform one more step in scalar code before moving to the vector unit.
   if (BuildVectorSDNode *BV =
           dyn_cast<BuildVectorSDNode>(N->getOperand(0)->getOperand(1))) {
     // Bail out if the vector isn't a constant splat.
     if (!BV->getConstantSplatNode())
       return SDValue();
 
     // Everything checks out. Build up the new and improved node.
     SDLoc DL(N);
     EVT IntVT = BV->getValueType(0);
     // Create a new constant of the appropriate type for the transformed
     // DAG.
     SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
     // The AND node needs bitcasts to/from an integer vector type around it.
     SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
     SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
                                  N->getOperand(0)->getOperand(0), MaskConst);
     SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
     return Res;
   }
 
   return SDValue();
 }

static SDValue PerformVZEXT_MOVLCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 21528 of file X86ISelLowering.cpp.

                                                                       {
   SDValue Op = N->getOperand(0);
   if (Op.getOpcode() == ISD::BITCAST)
     Op = Op.getOperand(0);
   EVT VT = N->getValueType(0), OpVT = Op.getValueType();
   if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
       VT.getVectorElementType().getSizeInBits() ==
       OpVT.getVectorElementType().getSizeInBits()) {
     return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
   }
   return SDValue();
 }

static SDValue performVZEXTCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

performVZEXTCombine - Performs build vector combines

Definition at line 22046 of file X86ISelLowering.cpp.

                                                                        {
   // (vzext (bitcast (vzext (x)) -> (vzext x)
   SDValue In = N->getOperand(0);
   while (In.getOpcode() == ISD::BITCAST)
     In = In.getOperand(0);
 
   if (In.getOpcode() != X86ISD::VZEXT)
     return SDValue();
 
   return DAG.getNode(X86ISD::VZEXT, SDLoc(N), N->getValueType(0),
                      In.getOperand(0));
 }

static SDValue PerformXorCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 20850 of file X86ISelLowering.cpp.

                                                                 {
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
   if (Subtarget->hasCMov()) {
     SDValue RV = performIntegerAbsCombine(N, DAG);
     if (RV.getNode())
       return RV;
   }
 
   return SDValue();
 }

static SDValue PerformZExtCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 21635 of file X86ISelLowering.cpp.

                                                                  {
   // (i32 zext (and (i8  x86isd::setcc_carry), 1)) ->
   //           (and (i32 x86isd::setcc_carry), 1)
   // This eliminates the zext. This transformation is necessary because
   // ISD::SETCC is always legalized to i8.
   SDLoc dl(N);
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
 
   if (N0.getOpcode() == ISD::AND &&
       N0.hasOneUse() &&
       N0.getOperand(0).hasOneUse()) {
     SDValue N00 = N0.getOperand(0);
     if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
       ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
       if (!C || C->getZExtValue() != 1)
         return SDValue();
       return DAG.getNode(ISD::AND, dl, VT,
                          DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
                                      N00.getOperand(0), N00.getOperand(1)),
                          DAG.getConstant(1, VT));
     }
   }
 
   if (N0.getOpcode() == ISD::TRUNCATE &&
       N0.hasOneUse() &&
       N0.getOperand(0).hasOneUse()) {
     SDValue N00 = N0.getOperand(0);
     if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
       return DAG.getNode(ISD::AND, dl, VT,
                          DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
                                      N00.getOperand(0), N00.getOperand(1)),
                          DAG.getConstant(1, VT));
     }
   }
   if (VT.is256BitVector()) {
     SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget);
     if (R.getNode())
       return R;
   }
 
   return SDValue();
 }

static SDValue PromoteSplat	(	ShuffleVectorSDNode *	SV,
		SelectionDAG &	DAG
	)

static

PromoteSplat - Splat is promoted to target supported vector shuffles.

Definition at line 5048 of file X86ISelLowering.cpp.

                                                                         {
   MVT SrcVT = SV->getSimpleValueType(0);
   SDValue V1 = SV->getOperand(0);
   SDLoc dl(SV);
 
   int EltNo = SV->getSplatIndex();
   int NumElems = SrcVT.getVectorNumElements();
   bool Is256BitVec = SrcVT.is256BitVector();
 
   assert(((SrcVT.is128BitVector() && NumElems > 4) || Is256BitVec) &&
          "Unknown how to promote splat for type");
 
   // Extract the 128-bit part containing the splat element and update
   // the splat element index when it refers to the higher register.
   if (Is256BitVec) {
     V1 = Extract128BitVector(V1, EltNo, DAG, dl);
     if (EltNo >= NumElems/2)
       EltNo -= NumElems/2;
   }
 
   // All i16 and i8 vector types can't be used directly by a generic shuffle
   // instruction because the target has no such instruction. Generate shuffles
   // which repeat i16 and i8 several times until they fit in i32, and then can
   // be manipulated by target suported shuffles.
   MVT EltVT = SrcVT.getVectorElementType();
   if (EltVT == MVT::i8 || EltVT == MVT::i16)
     V1 = PromoteSplati8i16(V1, DAG, EltNo);
 
   // Recreate the 256-bit vector and place the same 128-bit vector
   // into the low and high part. This is necessary because we want
   // to use VPERM* to shuffle the vectors
   if (Is256BitVec) {
     V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, V1, V1);
   }
 
   return getLegalSplat(DAG, V1, EltNo);
 }

static SDValue PromoteSplati8i16	(	SDValue	V,
		SelectionDAG &	DAG,
		int &	EltNo
	)

static

Definition at line 5004 of file X86ISelLowering.cpp.

                                                                            {
   MVT VT = V.getSimpleValueType();
   int NumElems = VT.getVectorNumElements();
   SDLoc dl(V);
 
   while (NumElems > 4) {
     if (EltNo < NumElems/2) {
       V = getUnpackl(DAG, dl, VT, V, V);
     } else {
       V = getUnpackh(DAG, dl, VT, V, V);
       EltNo -= NumElems/2;
     }
     NumElems >>= 1;
   }
   return V;
 }

static void ReplaceATOMIC_LOAD	(	SDNode *	Node,
		SmallVectorImpl< SDValue > &	Results,
		SelectionDAG &	DAG
	)

static

Definition at line 16295 of file X86ISelLowering.cpp.

                                                   {
   SDLoc dl(Node);
   EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
 
   // Convert wide load -> cmpxchg8b/cmpxchg16b
   // FIXME: On 32-bit, load -> fild or movq would be more efficient
   //        (The only way to get a 16-byte load is cmpxchg16b)
   // FIXME: 16-byte ATOMIC_CMP_SWAP isn't actually hooked up at the moment.
   SDValue Zero = DAG.getConstant(0, VT);
   SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
   SDValue Swap =
       DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, VT, VTs,
                            Node->getOperand(0), Node->getOperand(1), Zero, Zero,
                            cast<AtomicSDNode>(Node)->getMemOperand(),
                            cast<AtomicSDNode>(Node)->getOrdering(),
                            cast<AtomicSDNode>(Node)->getOrdering(),
                            cast<AtomicSDNode>(Node)->getSynchScope());
   Results.push_back(Swap.getValue(0));
   Results.push_back(Swap.getValue(2));
 }

static SDValue RewriteAsNarrowerShuffle	(	ShuffleVectorSDNode *	SVOp,
		SelectionDAG &	DAG
	)

static

RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide ones, or rewriting v4i32 / v4f32 as 2 wide ones if possible. This can be done when every pair / quad of shuffle mask elements point to elements in the right sequence. e.g. vector_shuffle X, Y, <2, 3, | 10, 11, | 0, 1, | 14, 15>

Definition at line 8562 of file X86ISelLowering.cpp.

                                                     {
   MVT VT = SVOp->getSimpleValueType(0);
   SDLoc dl(SVOp);
   unsigned NumElems = VT.getVectorNumElements();
   MVT NewVT;
   unsigned Scale;
   switch (VT.SimpleTy) {
   default: llvm_unreachable("Unexpected!");
   case MVT::v2i64:
   case MVT::v2f64:
            return SDValue(SVOp, 0);
   case MVT::v4f32:  NewVT = MVT::v2f64; Scale = 2; break;
   case MVT::v4i32:  NewVT = MVT::v2i64; Scale = 2; break;
   case MVT::v8i16:  NewVT = MVT::v4i32; Scale = 2; break;
   case MVT::v16i8:  NewVT = MVT::v4i32; Scale = 4; break;
   case MVT::v16i16: NewVT = MVT::v8i32; Scale = 2; break;
   case MVT::v32i8:  NewVT = MVT::v8i32; Scale = 4; break;
   }
 
   SmallVector<int, 8> MaskVec;
   for (unsigned i = 0; i != NumElems; i += Scale) {
     int StartIdx = -1;
     for (unsigned j = 0; j != Scale; ++j) {
       int EltIdx = SVOp->getMaskElt(i+j);
       if (EltIdx < 0)
         continue;
       if (StartIdx < 0)
         StartIdx = (EltIdx / Scale);
       if (EltIdx != (int)(StartIdx*Scale + j))
         return SDValue();
     }
     MaskVec.push_back(StartIdx);
   }
 
   SDValue V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(0));
   SDValue V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(1));
   return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]);
 }

static bool shouldLowerAsInterleaving ( ArrayRef< int > Mask )

static

Detect whether the mask pattern should be lowered through interleaving.

This essentially tests whether viewing the mask as an interleaving of two sub-sequences reduces the cross-input traffic of a blend operation. If so, lowering it through interleaving is a significantly better strategy.

Definition at line 7434 of file X86ISelLowering.cpp.

                                                           {
   int NumEvenInputs[2] = {0, 0};
   int NumOddInputs[2] = {0, 0};
   int NumLoInputs[2] = {0, 0};
   int NumHiInputs[2] = {0, 0};
   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
     if (Mask[i] < 0)
       continue;
 
     int InputIdx = Mask[i] >= Size;
 
     if (i < Size / 2)
       ++NumLoInputs[InputIdx];
     else
       ++NumHiInputs[InputIdx];
 
     if ((i % 2) == 0)
       ++NumEvenInputs[InputIdx];
     else
       ++NumOddInputs[InputIdx];
   }
 
   // The minimum number of cross-input results for both the interleaved and
   // split cases. If interleaving results in fewer cross-input results, return
   // true.
   int InterleavedCrosses = std::min(NumEvenInputs[1] + NumOddInputs[0],
                                     NumEvenInputs[0] + NumOddInputs[1]);
   int SplitCrosses = std::min(NumLoInputs[1] + NumHiInputs[0],
                               NumLoInputs[0] + NumHiInputs[1]);
   return InterleavedCrosses < SplitCrosses;
 }

static bool ShouldXformToMOVHLPS	(	ArrayRef< int >	Mask,
		MVT	VT
	)

static

ShouldXformToMOVHLPS - Return true if the node should be transformed to match movhlps. The lower half elements should come from upper half of V1 (and in order), and the upper half elements should come from the upper half of V2 (and in order).

Definition at line 4774 of file X86ISelLowering.cpp.

                                                              {
   if (!VT.is128BitVector())
     return false;
   if (VT.getVectorNumElements() != 4)
     return false;
   for (unsigned i = 0, e = 2; i != e; ++i)
     if (!isUndefOrEqual(Mask[i], i+2))
       return false;
   for (unsigned i = 2; i != 4; ++i)
     if (!isUndefOrEqual(Mask[i], i+4))
       return false;
   return true;
 }

static bool ShouldXformToMOVLP	(	SDNode *	V1,
		SDNode *	V2,
		ArrayRef< int >	Mask,
		MVT	VT
	)

static

ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to match movlp{s|d}. The lower half elements should come from lower half of V1 (and in order), and the upper half elements should come from the upper half of V2 (and in order). And since V1 will become the source of the MOVLP, it must be either a vector load or a scalar load to vector.

Definition at line 4830 of file X86ISelLowering.cpp.

                                                            {
   if (!VT.is128BitVector())
     return false;
 
   if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
     return false;
   // Is V2 is a vector load, don't do this transformation. We will try to use
   // load folding shufps op.
   if (ISD::isNON_EXTLoad(V2) || WillBeConstantPoolLoad(V2))
     return false;
 
   unsigned NumElems = VT.getVectorNumElements();
 
   if (NumElems != 2 && NumElems != 4)
     return false;
   for (unsigned i = 0, e = NumElems/2; i != e; ++i)
     if (!isUndefOrEqual(Mask[i], i))
       return false;
   for (unsigned i = NumElems/2, e = NumElems; i != e; ++i)
     if (!isUndefOrEqual(Mask[i], i+NumElems))
       return false;
   return true;
 }

static bool ShuffleCrosses128bitLane	(	MVT	VT,
		unsigned	InputIdx,
		unsigned	OutputIdx
	)

static

In vector type VT, return true if the element at index InputIdx falls on a different 128-bit lane than OutputIdx.

Definition at line 8086 of file X86ISelLowering.cpp.

                                                          {
   unsigned EltSize = VT.getVectorElementType().getSizeInBits();
   return InputIdx * EltSize / 128 != OutputIdx * EltSize / 128;
 }

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"
	)

static SDValue TransformVSELECTtoBlendVECTOR_SHUFFLE	(	SDNode *	N,
		SelectionDAG &	DAG,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 19206 of file X86ISelLowering.cpp.

                                                                      {
   SDLoc dl(N);
   SDValue Cond = N->getOperand(0);
   SDValue LHS = N->getOperand(1);
   SDValue RHS = N->getOperand(2);
 
   if (Cond.getOpcode() == ISD::SIGN_EXTEND) {
     SDValue CondSrc = Cond->getOperand(0);
     if (CondSrc->getOpcode() == ISD::SIGN_EXTEND_INREG)
       Cond = CondSrc->getOperand(0);
   }
 
   MVT VT = N->getSimpleValueType(0);
   MVT EltVT = VT.getVectorElementType();
   unsigned NumElems = VT.getVectorNumElements();
   // There is no blend with immediate in AVX-512.
   if (VT.is512BitVector())
     return SDValue();
 
   if (!Subtarget->hasSSE41() || EltVT == MVT::i8)
     return SDValue();
   if (!Subtarget->hasInt256() && VT == MVT::v16i16)
     return SDValue();
 
   if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
     return SDValue();
 
   unsigned MaskValue = 0;
   if (!BUILD_VECTORtoBlendMask(cast<BuildVectorSDNode>(Cond), MaskValue))
     return SDValue();
 
   SmallVector<int, 8> ShuffleMask(NumElems, -1);
   for (unsigned i = 0; i < NumElems; ++i) {
     // Be sure we emit undef where we can.
     if (Cond.getOperand(i)->getOpcode() == ISD::UNDEF)
       ShuffleMask[i] = -1;
     else
       ShuffleMask[i] = i + NumElems * ((MaskValue >> i) & 1);
   }
 
   return DAG.getVectorShuffle(VT, dl, LHS, RHS, &ShuffleMask[0]);
 }

static unsigned TranslateX86CC	(	ISD::CondCode	SetCCOpcode,
		bool	isFP,
		SDValue &	LHS,
		SDValue &	RHS,
		SelectionDAG &	DAG
	)

static

TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 specific condition code, returning the condition code and the LHS/RHS of the comparison to make.

Definition at line 3555 of file X86ISelLowering.cpp.

                                                                               {
   if (!isFP) {
     if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
       if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
         // X > -1   -> X == 0, jump !sign.
         RHS = DAG.getConstant(0, RHS.getValueType());
         return X86::COND_NS;
       }
       if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
         // X < 0   -> X == 0, jump on sign.
         return X86::COND_S;
       }
       if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) {
         // X < 1   -> X <= 0
         RHS = DAG.getConstant(0, RHS.getValueType());
         return X86::COND_LE;
       }
     }
 
     switch (SetCCOpcode) {
     default: llvm_unreachable("Invalid integer condition!");
     case ISD::SETEQ:  return X86::COND_E;
     case ISD::SETGT:  return X86::COND_G;
     case ISD::SETGE:  return X86::COND_GE;
     case ISD::SETLT:  return X86::COND_L;
     case ISD::SETLE:  return X86::COND_LE;
     case ISD::SETNE:  return X86::COND_NE;
     case ISD::SETULT: return X86::COND_B;
     case ISD::SETUGT: return X86::COND_A;
     case ISD::SETULE: return X86::COND_BE;
     case ISD::SETUGE: return X86::COND_AE;
     }
   }
 
   // First determine if it is required or is profitable to flip the operands.
 
   // If LHS is a foldable load, but RHS is not, flip the condition.
   if (ISD::isNON_EXTLoad(LHS.getNode()) &&
       !ISD::isNON_EXTLoad(RHS.getNode())) {
     SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
     std::swap(LHS, RHS);
   }
 
   switch (SetCCOpcode) {
   default: break;
   case ISD::SETOLT:
   case ISD::SETOLE:
   case ISD::SETUGT:
   case ISD::SETUGE:
     std::swap(LHS, RHS);
     break;
   }
 
   // On a floating point condition, the flags are set as follows:
   // ZF  PF  CF   op
   //  0 | 0 | 0 | X > Y
   //  0 | 0 | 1 | X < Y
   //  1 | 0 | 0 | X == Y
   //  1 | 1 | 1 | unordered
   switch (SetCCOpcode) {
   default: llvm_unreachable("Condcode should be pre-legalized away");
   case ISD::SETUEQ:
   case ISD::SETEQ:   return X86::COND_E;
   case ISD::SETOLT:              // flipped
   case ISD::SETOGT:
   case ISD::SETGT:   return X86::COND_A;
   case ISD::SETOLE:              // flipped
   case ISD::SETOGE:
   case ISD::SETGE:   return X86::COND_AE;
   case ISD::SETUGT:              // flipped
   case ISD::SETULT:
   case ISD::SETLT:   return X86::COND_B;
   case ISD::SETUGE:              // flipped
   case ISD::SETULE:
   case ISD::SETLE:   return X86::COND_BE;
   case ISD::SETONE:
   case ISD::SETNE:   return X86::COND_NE;
   case ISD::SETUO:   return X86::COND_P;
   case ISD::SETO:    return X86::COND_NP;
   case ISD::SETOEQ:
   case ISD::SETUNE:  return X86::COND_INVALID;
   }
 }

static int translateX86FSETCC	(	ISD::CondCode	SetCCOpcode,
		SDValue &	Op0,
		SDValue &	Op1
	)

static

Turns an ISD::CondCode into a value suitable for SSE floating point mask CMPs.

Definition at line 12027 of file X86ISelLowering.cpp.

                                             {
   unsigned SSECC;
   bool Swap = false;
 
   // SSE Condition code mapping:
   //  0 - EQ
   //  1 - LT
   //  2 - LE
   //  3 - UNORD
   //  4 - NEQ
   //  5 - NLT
   //  6 - NLE
   //  7 - ORD
   switch (SetCCOpcode) {
   default: llvm_unreachable("Unexpected SETCC condition");
   case ISD::SETOEQ:
   case ISD::SETEQ:  SSECC = 0; break;
   case ISD::SETOGT:
   case ISD::SETGT:  Swap = true; // Fallthrough
   case ISD::SETLT:
   case ISD::SETOLT: SSECC = 1; break;
   case ISD::SETOGE:
   case ISD::SETGE:  Swap = true; // Fallthrough
   case ISD::SETLE:
   case ISD::SETOLE: SSECC = 2; break;
   case ISD::SETUO:  SSECC = 3; break;
   case ISD::SETUNE:
   case ISD::SETNE:  SSECC = 4; break;
   case ISD::SETULE: Swap = true; // Fallthrough
   case ISD::SETUGE: SSECC = 5; break;
   case ISD::SETULT: Swap = true; // Fallthrough
   case ISD::SETUGT: SSECC = 6; break;
   case ISD::SETO:   SSECC = 7; break;
   case ISD::SETUEQ:
   case ISD::SETONE: SSECC = 8; break;
   }
   if (Swap)
     std::swap(Op0, Op1);
 
   return SSECC;
 }

static SDValue WidenMaskArithmetic	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 20511 of file X86ISelLowering.cpp.

                                                                 {
   EVT VT = N->getValueType(0);
   if (!VT.is256BitVector())
     return SDValue();
 
   assert((N->getOpcode() == ISD::ANY_EXTEND ||
           N->getOpcode() == ISD::ZERO_EXTEND ||
           N->getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node");
 
   SDValue Narrow = N->getOperand(0);
   EVT NarrowVT = Narrow->getValueType(0);
   if (!NarrowVT.is128BitVector())
     return SDValue();
 
   if (Narrow->getOpcode() != ISD::XOR &&
       Narrow->getOpcode() != ISD::AND &&
       Narrow->getOpcode() != ISD::OR)
     return SDValue();
 
   SDValue N0  = Narrow->getOperand(0);
   SDValue N1  = Narrow->getOperand(1);
   SDLoc DL(Narrow);
 
   // The Left side has to be a trunc.
   if (N0.getOpcode() != ISD::TRUNCATE)
     return SDValue();
 
   // The type of the truncated inputs.
   EVT WideVT = N0->getOperand(0)->getValueType(0);
   if (WideVT != VT)
     return SDValue();
 
   // The right side has to be a 'trunc' or a constant vector.
   bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE;
   ConstantSDNode *RHSConstSplat = nullptr;
   if (auto *RHSBV = dyn_cast<BuildVectorSDNode>(N1))
     RHSConstSplat = RHSBV->getConstantSplatNode();
   if (!RHSTrunc && !RHSConstSplat)
     return SDValue();
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   if (!TLI.isOperationLegalOrPromote(Narrow->getOpcode(), WideVT))
     return SDValue();
 
   // Set N0 and N1 to hold the inputs to the new wide operation.
   N0 = N0->getOperand(0);
   if (RHSConstSplat) {
     N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT.getScalarType(),
                      SDValue(RHSConstSplat, 0));
     SmallVector<SDValue, 8> C(WideVT.getVectorNumElements(), N1);
     N1 = DAG.getNode(ISD::BUILD_VECTOR, DL, WideVT, C);
   } else if (RHSTrunc) {
     N1 = N1->getOperand(0);
   }
 
   // Generate the wide operation.
   SDValue Op = DAG.getNode(Narrow->getOpcode(), DL, WideVT, N0, N1);
   unsigned Opcode = N->getOpcode();
   switch (Opcode) {
   case ISD::ANY_EXTEND:
     return Op;
   case ISD::ZERO_EXTEND: {
     unsigned InBits = NarrowVT.getScalarType().getSizeInBits();
     APInt Mask = APInt::getAllOnesValue(InBits);
     Mask = Mask.zext(VT.getScalarType().getSizeInBits());
     return DAG.getNode(ISD::AND, DL, VT,
                        Op, DAG.getConstant(Mask, VT));
   }
   case ISD::SIGN_EXTEND:
     return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
                        Op, DAG.getValueType(NarrowVT));
   default:
     llvm_unreachable("Unexpected opcode");
   }
 }

static bool WillBeConstantPoolLoad ( SDNode * N )

static

Definition at line 4804 of file X86ISelLowering.cpp.

                                               {
   if (N->getOpcode() != ISD::BUILD_VECTOR)
     return false;
 
   // Check for any non-constant elements.
   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
     switch (N->getOperand(i).getNode()->getOpcode()) {
     case ISD::UNDEF:
     case ISD::ConstantFP:
     case ISD::Constant:
       break;
     default:
       return false;
     }
 
   // Vectors of all-zeros and all-ones are materialized with special
   // instructions rather than being loaded.
   return !ISD::isBuildVectorAllZeros(N) &&
          !ISD::isBuildVectorAllOnes(N);
 }

static SDValue XFormVExtractWithShuffleIntoLoad	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI
	)

static

XFormVExtractWithShuffleIntoLoad - Check if a vector extract from a target specific shuffle of a load can be folded into a single element load. Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but shuffles have been customed lowered so we need to handle those here.

Definition at line 18942 of file X86ISelLowering.cpp.

                                                                              {
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
   SDValue InVec = N->getOperand(0);
   SDValue EltNo = N->getOperand(1);
 
   if (!isa<ConstantSDNode>(EltNo))
     return SDValue();
 
   EVT VT = InVec.getValueType();
 
   bool HasShuffleIntoBitcast = false;
   if (InVec.getOpcode() == ISD::BITCAST) {
     // Don't duplicate a load with other uses.
     if (!InVec.hasOneUse())
       return SDValue();
     EVT BCVT = InVec.getOperand(0).getValueType();
     if (BCVT.getVectorNumElements() != VT.getVectorNumElements())
       return SDValue();
     InVec = InVec.getOperand(0);
     HasShuffleIntoBitcast = true;
   }
 
   if (!isTargetShuffle(InVec.getOpcode()))
     return SDValue();
 
   // Don't duplicate a load with other uses.
   if (!InVec.hasOneUse())
     return SDValue();
 
   SmallVector<int, 16> ShuffleMask;
   bool UnaryShuffle;
   if (!getTargetShuffleMask(InVec.getNode(), VT.getSimpleVT(), ShuffleMask,
                             UnaryShuffle))
     return SDValue();
 
   // Select the input vector, guarding against out of range extract vector.
   unsigned NumElems = VT.getVectorNumElements();
   int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
   int Idx = (Elt > (int)NumElems) ? -1 : ShuffleMask[Elt];
   SDValue LdNode = (Idx < (int)NumElems) ? InVec.getOperand(0)
                                          : InVec.getOperand(1);
 
   // If inputs to shuffle are the same for both ops, then allow 2 uses
   unsigned AllowedUses = InVec.getOperand(0) == InVec.getOperand(1) ? 2 : 1;
 
   if (LdNode.getOpcode() == ISD::BITCAST) {
     // Don't duplicate a load with other uses.
     if (!LdNode.getNode()->hasNUsesOfValue(AllowedUses, 0))
       return SDValue();
 
     AllowedUses = 1; // only allow 1 load use if we have a bitcast
     LdNode = LdNode.getOperand(0);
   }
 
   if (!ISD::isNormalLoad(LdNode.getNode()))
     return SDValue();
 
   LoadSDNode *LN0 = cast<LoadSDNode>(LdNode);
 
   if (!LN0 ||!LN0->hasNUsesOfValue(AllowedUses, 0) || LN0->isVolatile())
     return SDValue();
 
   if (HasShuffleIntoBitcast) {
     // If there's a bitcast before the shuffle, check if the load type and
     // alignment is valid.
     unsigned Align = LN0->getAlignment();
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     unsigned NewAlign = TLI.getDataLayout()->
       getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
 
     if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VT))
       return SDValue();
   }
 
   // All checks match so transform back to vector_shuffle so that DAG combiner
   // can finish the job
   SDLoc dl(N);
 
   // Create shuffle node taking into account the case that its a unary shuffle
   SDValue Shuffle = (UnaryShuffle) ? DAG.getUNDEF(VT) : InVec.getOperand(1);
   Shuffle = DAG.getVectorShuffle(InVec.getValueType(), dl,
                                  InVec.getOperand(0), Shuffle,
                                  &ShuffleMask[0]);
   Shuffle = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), Shuffle,
                      EltNo);
 }

Variable Documentation

cl::opt<bool> ExperimentalVectorShuffleLowering("x86-experimental-vector-shuffle-lowering", cl::init(false), cl::desc("Enable an experimental vector shuffle lowering code path."), cl::Hidden)

static

cl::opt<bool> ExperimentalVectorWideningLegalization("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening ""rather than promotion."), cl::Hidden)

static

std::map< unsigned, IntrinsicData> IntrMap

Definition at line 14397 of file X86ISelLowering.cpp.

Data Structures

Macros

Enumerations

Functions

Variables

Macro Definition Documentation

Enumeration Type Documentation

Function Documentation

Variable Documentation